You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/11/06 22:01:35 UTC
[01/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/yarn-native-services 42f5fed21 -> a55d0738f (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
deleted file mode 100644
index 8310530..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test loading example resources.
- */
-@RunWith(value = Parameterized.class)
-public class TestLoadExampleAppJson extends Assert {
- private String resource;
-
- public TestLoadExampleAppJson(String resource) {
- this.resource = resource;
- }
-
- @Parameterized.Parameters
- public static Collection<String[]> filenames() {
- String[][] stringArray = new String[ExampleAppJson
- .ALL_EXAMPLE_RESOURCES.size()][1];
- int i = 0;
- for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) {
- stringArray[i++][0] = s;
- }
- return Arrays.asList(stringArray);
- }
-
- @Test
- public void testLoadResource() throws Throwable {
- try {
- Application application = JSON_SER_DESER.fromResource(resource);
-
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
-
- ServiceApiUtil.validateAndResolveApplication(application, sfs,
- new YarnConfiguration());
- } catch (Exception e) {
- throw new Exception("exception loading " + resource + ":" + e.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
deleted file mode 100644
index 98c78d3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Test cluster name validation.
- */
-public class TestValidateServiceNames {
-
- void assertValidName(String name) {
- boolean valid = SliderUtils.isClusternameValid(name);
- Assert.assertTrue("Clustername '" + name + "' mistakenly declared invalid",
- valid);
- }
-
- void assertInvalidName(String name) {
- boolean valid = SliderUtils.isClusternameValid(name);
- Assert.assertFalse("Clustername '\" + name + \"' mistakenly declared valid",
- valid);
- }
-
- void assertInvalid(List<String> names) {
- for (String name : names) {
- assertInvalidName(name);
- }
- }
-
- void assertValid(List<String> names) {
- for (String name : names) {
- assertValidName(name);
- }
- }
-
- @Test
- public void testEmptyName() throws Throwable {
- assertInvalidName("");
- }
-
- @Test
- public void testSpaceName() throws Throwable {
- assertInvalidName(" ");
- }
-
-
- @Test
- public void testLeadingHyphen() throws Throwable {
- assertInvalidName("-hyphen");
- }
-
- @Test
- public void testTitleLetters() throws Throwable {
- assertInvalidName("Title");
- }
-
- @Test
- public void testCapitalLetters() throws Throwable {
- assertInvalidName("UPPER-CASE-CLUSTER");
- }
-
- @Test
- public void testInnerBraced() throws Throwable {
- assertInvalidName("a[a");
- }
-
- @Test
- public void testLeadingBrace() throws Throwable {
- assertInvalidName("[");
- }
-
- @Test
- public void testNonalphaLeadingChars() throws Throwable {
- assertInvalid(Arrays.asList(
- "[a", "#", "@", "=", "*", "."
- ));
- }
-
- @Test
- public void testNonalphaInnerChars() throws Throwable {
- assertInvalid(Arrays.asList(
- "a[a", "b#", "c@", "d=", "e*", "f.", "g ", "h i"
- ));
- }
-
- @Test
- public void testClusterValid() throws Throwable {
- assertValidName("cluster");
- }
-
- @Test
- public void testValidNames() throws Throwable {
- assertValid(Arrays.asList(
- "cluster",
- "cluster1",
- "very-very-very-long-cluster-name",
- "c1234567890"
- ));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
deleted file mode 100644
index 5b24a1d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.providers;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.easymock.EasyMock.*;
-
-/**
- * Test the AbstractClientProvider shared methods.
- */
-public class TestAbstractClientProvider {
- private static final String EXCEPTION_PREFIX = "Should have thrown " +
- "exception: ";
- private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
- "exception: ";
-
- private static class ClientProvider extends AbstractClientProvider {
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fileSystem)
- throws IOException {
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile,
- FileSystem fileSystem) throws IOException {
- }
- }
-
- @Test
- public void testConfigFiles() throws IOException {
- ClientProvider clientProvider = new ClientProvider();
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
- replay(mockFs);
-
- ConfigFile configFile = new ConfigFile();
- List<ConfigFile> configFiles = new ArrayList<>();
- configFiles.add(configFile);
-
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "null file type");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setType(ConfigFile.TypeEnum.TEMPLATE);
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setSrcFile("srcfile");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "empty dest file");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setDestFile("destfile");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- configFile = new ConfigFile();
- configFile.setType(ConfigFile.TypeEnum.JSON);
- configFile.setSrcFile(null);
- configFile.setDestFile("path/destfile2");
- configFiles.add(configFile);
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "dest file with multiple path elements");
- } catch (IllegalArgumentException e) {
- }
-
- configFile.setDestFile("/path/destfile2");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- configFile.setDestFile("destfile");
- try {
- clientProvider.validateConfigFiles(configFiles, mockFs);
- Assert.fail(EXCEPTION_PREFIX + "duplicate dest file");
- } catch (IllegalArgumentException e) {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
deleted file mode 100644
index 489578d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.providers;
-
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultClientProvider;
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderService;
-import org.apache.hadoop.yarn.service.provider.docker.DockerClientProvider;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderService;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballClientProvider;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderService;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test provider factories.
- */
-public class TestProviderFactory {
- @Test
- public void testDockerFactory() throws Throwable {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(new Artifact().type(TypeEnum.DOCKER));
- assertTrue(factory instanceof DockerProviderFactory);
- assertTrue(factory.createClientProvider() instanceof DockerClientProvider);
- assertTrue(factory.createServerProvider() instanceof DockerProviderService);
- assertTrue(ProviderFactory.getProviderService(new Artifact()
- .type(TypeEnum.DOCKER)) instanceof DockerProviderService);
- }
-
- @Test
- public void testTarballFactory() throws Throwable {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(new Artifact().type(TypeEnum.TARBALL));
- assertTrue(factory instanceof TarballProviderFactory);
- assertTrue(factory.createClientProvider() instanceof TarballClientProvider);
- assertTrue(factory.createServerProvider() instanceof
- TarballProviderService);
- assertTrue(ProviderFactory.getProviderService(new Artifact()
- .type(TypeEnum.TARBALL)) instanceof TarballProviderService);
- }
-
- @Test
- public void testDefaultFactory() throws Throwable {
- ProviderFactory factory = ProviderFactory
- .createSliderProviderFactory(null);
- assertTrue(factory instanceof DefaultProviderFactory);
- assertTrue(factory.createClientProvider() instanceof DefaultClientProvider);
- assertTrue(factory.createServerProvider() instanceof DefaultProviderService);
- assertTrue(ProviderFactory.getProviderService(null) instanceof
- DefaultProviderService);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
deleted file mode 100644
index 6f5653f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/servicemonitor/TestServiceMonitor.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.yarn.service.servicemonitor;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.MockServiceAM;
-import org.apache.hadoop.yarn.service.ServiceTestUtils;
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-
-public class TestServiceMonitor extends ServiceTestUtils {
-
- private File basedir;
- YarnConfiguration conf = new YarnConfiguration();
-
- @Before
- public void setup() throws Exception {
- basedir = new File("target", "apps");
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
- conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 2);
- }
-
- @After
- public void tearDown() throws IOException {
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- }
-
- // Create compa with 1 container
- // Create compb with 1 container
- // Verify compb dependency satisfied
- // Increase compa to 2 containers
- // Verify compb dependency becomes unsatisfied.
- @Test
- public void testComponentDependency() throws Exception{
- ApplicationId applicationId = ApplicationId.newInstance(123456, 1);
- Application exampleApp = new Application();
- exampleApp.setId(applicationId.toString());
- exampleApp.setName("testComponentDependency");
- exampleApp.addComponent(createComponent("compa", 1, "sleep 1000"));
- Component compb = createComponent("compb", 1, "sleep 1000");
-
- // Let compb depends on compa;
- compb.setDependencies(Collections.singletonList("compa"));
- exampleApp.addComponent(compb);
-
- MockServiceAM am = new MockServiceAM(exampleApp);
- am.init(conf);
- am.start();
-
- // compa ready
- Assert.assertTrue(am.getComponent("compa").areDependenciesReady());
- //compb not ready
- Assert.assertFalse(am.getComponent("compb").areDependenciesReady());
-
- // feed 1 container to compa,
- am.feedContainerToComp(exampleApp, 1, "compa");
- // waiting for compb's dependencies are satisfied
- am.waitForDependenciesSatisfied("compb");
-
- // feed 1 container to compb
- am.feedContainerToComp(exampleApp, 2, "compb");
- am.flexComponent("compa", 2);
- am.waitForNumDesiredContainers("compa", 2);
-
- // compb dependencies not satisfied again.
- Assert.assertFalse(am.getComponent("compb").areDependenciesReady());
- am.stop();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
deleted file mode 100644
index a891df8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ApplicationState;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Test class for ServiceTimelinePublisher.
- */
-public class TestServiceTimelinePublisher {
- private TimelineV2Client timelineClient;
- private Configuration config;
- private ServiceTimelinePublisher serviceTimelinePublisher;
- private static String SERVICE_NAME = "HBASE";
- private static String SERVICEID = "application_1490093646524_0005";
- private static String ARTIFACTID = "ARTIFACTID";
- private static String COMPONENT_NAME = "DEFAULT";
- private static String CONTAINER_ID =
- "container_e02_1490093646524_0005_01_000001";
- private static String CONTAINER_IP =
- "localhost";
- private static String CONTAINER_HOSTNAME =
- "cnl124-localhost.site";
- private static String CONTAINER_BAREHOST =
- "localhost.com";
-
- @Before
- public void setUp() throws Exception {
- config = new Configuration();
- config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
- timelineClient =
- new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
- serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
- timelineClient.init(config);
- serviceTimelinePublisher.init(config);
- timelineClient.start();
- serviceTimelinePublisher.start();
- }
-
- @After
- public void tearDown() throws Exception {
- if (serviceTimelinePublisher != null) {
- serviceTimelinePublisher.stop();
- }
- if (timelineClient != null) {
- timelineClient.stop();
- }
- }
-
- @Test
- public void testServiceAttemptEntity() {
- Application application = createMockApplication();
- serviceTimelinePublisher
- .serviceAttemptRegistered(application, new YarnConfiguration());
-
- Collection<TimelineEntity> lastPublishedEntities =
- ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
- // 2 entities because during registration component also registered.
- assertEquals(2, lastPublishedEntities.size());
- for (TimelineEntity timelineEntity : lastPublishedEntities) {
- if (timelineEntity.getType() == ServiceTimelineEntityType.COMPONENT
- .toString()) {
- verifyComponentTimelineEntity(timelineEntity);
- } else {
- verifyServiceAttemptTimelineEntity(timelineEntity, null, true);
- }
- }
-
- ServiceContext context = new ServiceContext();
- context.attemptId = ApplicationAttemptId
- .newInstance(ApplicationId.fromString(application.getId()), 1);
- String exitDiags = "service killed";
- serviceTimelinePublisher.serviceAttemptUnregistered(context, exitDiags);
- lastPublishedEntities =
- ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
- for (TimelineEntity timelineEntity : lastPublishedEntities) {
- if (timelineEntity.getType() == ServiceTimelineEntityType.SERVICE_ATTEMPT
- .toString()) {
- verifyServiceAttemptTimelineEntity(timelineEntity, exitDiags,
- false);
- }
- }
- }
-
- @Test
- public void testComponentInstanceEntity() {
- Container container = new Container();
- container.id(CONTAINER_ID).ip(CONTAINER_IP).bareHost(CONTAINER_BAREHOST)
- .hostname(CONTAINER_HOSTNAME).state(ContainerState.RUNNING_BUT_UNREADY)
- .launchTime(new Date());
- ComponentInstanceId id = new ComponentInstanceId(0, COMPONENT_NAME);
- ComponentInstance instance = mock(ComponentInstance.class);
- when(instance.getCompName()).thenReturn(COMPONENT_NAME);
- when(instance.getCompInstanceName()).thenReturn("comp_instance_name");
- serviceTimelinePublisher.componentInstanceStarted(container,
- instance);
-
- Collection<TimelineEntity> lastPublishedEntities =
- ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
- assertEquals(1, lastPublishedEntities.size());
- TimelineEntity entity = lastPublishedEntities.iterator().next();
-
- assertEquals(1, entity.getEvents().size());
- assertEquals(CONTAINER_ID, entity.getId());
- assertEquals(CONTAINER_BAREHOST,
- entity.getInfo().get(ServiceTimelineMetricsConstants.BARE_HOST));
- assertEquals(COMPONENT_NAME,
- entity.getInfo().get(ServiceTimelineMetricsConstants.COMPONENT_NAME));
- assertEquals(ContainerState.RUNNING_BUT_UNREADY.toString(),
- entity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
-
- // updated container state
- container.setState(ContainerState.READY);
- serviceTimelinePublisher.componentInstanceUpdated(container);
- lastPublishedEntities =
- ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
- assertEquals(1, lastPublishedEntities.size());
- entity = lastPublishedEntities.iterator().next();
- assertEquals(2, entity.getEvents().size());
- assertEquals(ContainerState.READY.toString(),
- entity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
-
- }
-
- private void verifyServiceAttemptTimelineEntity(TimelineEntity timelineEntity,
- String message, boolean isRegistedEntity) {
- assertEquals(SERVICEID, timelineEntity.getId());
- assertEquals(SERVICE_NAME,
- timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.NAME));
- if (isRegistedEntity) {
- assertEquals(ApplicationState.STARTED.toString(),
- timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
- assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString(),
- timelineEntity.getEvents().iterator().next().getId());
- } else {
- assertEquals("ENDED",
- timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE).toString());
- assertEquals(message, timelineEntity.getInfo()
- .get(ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO));
- assertEquals(2, timelineEntity.getEvents().size());
- assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString(),
- timelineEntity.getEvents().iterator().next().getId());
- }
- }
-
- private void verifyComponentTimelineEntity(TimelineEntity entity) {
- Map<String, Object> info = entity.getInfo();
- assertEquals("DEFAULT", entity.getId());
- assertEquals(ARTIFACTID,
- info.get(ServiceTimelineMetricsConstants.ARTIFACT_ID));
- assertEquals("DOCKER",
- info.get(ServiceTimelineMetricsConstants.ARTIFACT_TYPE));
- assertEquals("medium",
- info.get(ServiceTimelineMetricsConstants.RESOURCE_PROFILE));
- assertEquals(1, info.get(ServiceTimelineMetricsConstants.RESOURCE_CPU));
- assertEquals("1024",
- info.get(ServiceTimelineMetricsConstants.RESOURCE_MEMORY));
- assertEquals("sleep 1",
- info.get(ServiceTimelineMetricsConstants.LAUNCH_COMMAND));
- assertEquals("false",
- info.get(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER));
- assertEquals("label",
- info.get(ServiceTimelineMetricsConstants.PLACEMENT_POLICY));
- }
-
- private static Application createMockApplication() {
- Application application = mock(Application.class);
-
- when(application.getId()).thenReturn(SERVICEID);
- when(application.getLaunchTime()).thenReturn(new Date());
- when(application.getState()).thenReturn(ApplicationState.STARTED);
- when(application.getName()).thenReturn(SERVICE_NAME);
- when(application.getConfiguration()).thenReturn(
- new org.apache.hadoop.yarn.service.api.records.Configuration());
-
- Component component = mock(Component.class);
- Artifact artifact = new Artifact();
- artifact.setId(ARTIFACTID);
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory(1024 + "");
- resource.setProfile("medium");
- when(component.getArtifact()).thenReturn(artifact);
- when(component.getName()).thenReturn(COMPONENT_NAME);
- when(component.getResource()).thenReturn(resource);
- when(component.getLaunchCommand()).thenReturn("sleep 1");
- PlacementPolicy placementPolicy = new PlacementPolicy();
- placementPolicy.setLabel("label");
- when(component.getPlacementPolicy()).thenReturn(placementPolicy);
- when(component.getConfiguration()).thenReturn(
- new org.apache.hadoop.yarn.service.api.records.Configuration());
- List<Component> components = new ArrayList<Component>();
- components.add(component);
-
- when(application.getComponents()).thenReturn(components);
- return application;
- }
-
- protected static class DummyTimelineClient extends TimelineV2ClientImpl {
- private Map<Identifier, TimelineEntity> lastPublishedEntities =
- new HashMap<>();
-
- public DummyTimelineClient(ApplicationId appId) {
- super(appId);
- }
-
- @Override
- public void putEntitiesAsync(TimelineEntity... entities)
- throws IOException, YarnException {
- putEntities(entities);
- }
-
- @Override
- public void putEntities(TimelineEntity... entities)
- throws IOException, YarnException {
- for (TimelineEntity timelineEntity : entities) {
- TimelineEntity entity =
- lastPublishedEntities.get(timelineEntity.getIdentifier());
- if (entity == null) {
- lastPublishedEntities.put(timelineEntity.getIdentifier(),
- timelineEntity);
- } else {
- entity.addMetrics(timelineEntity.getMetrics());
- entity.addEvents(timelineEntity.getEvents());
- entity.addInfo(timelineEntity.getInfo());
- entity.addConfigs(timelineEntity.getConfigs());
- entity.addRelatesToEntities(timelineEntity.getRelatesToEntities());
- entity
- .addIsRelatedToEntities(timelineEntity.getIsRelatedToEntities());
- }
- }
- }
-
- public Collection<TimelineEntity> getLastPublishedEntities() {
- return lastPublishedEntities.values();
- }
-
- public void reset() {
- lastPublishedEntities = null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/example-app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/example-app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/example-app.json
deleted file mode 100644
index 5dfbd64..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/example-app.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "name": "example-app",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/log4j.properties
deleted file mode 100644
index 3adbaa4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=INFO,stdout
-log4j.threshhold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-
-log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
-log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
-log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
-#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
-
-log4j.logger.org.apache.slider=DEBUG
-log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-log4j.logger.org.apache.hadoop.yarn.registry=DEBUG
-
-#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
-#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
-
-#crank back on some noise
-log4j.logger.org.apache.hadoop.ipc.CallQueueManager=WARN
-
-log4j.logger.org.apache.hadoop.util.Shell=ERROR
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-log4j.logger.org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager=FATAL
-org.apache.hadoop.security.authentication.server.AuthenticationFilter=WARN
-log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
-log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
-log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
-log4j.logger.org.apache.hadoop.hdfs=WARN
-log4j.logger.BlockStateChange=WARN
-
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
-
-log4j.logger.org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl=WARN
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitorImpl=ERROR
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher=WARN
-log4j.logger.org.apache.hadoop.metrics2=ERROR
-log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
-log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
-log4j.logger.org.apache.hadoop.security.token.delegation=WARN
-log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
deleted file mode 100644
index d7e2fd0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
+++ /dev/null
@@ -1,72 +0,0 @@
-{
- "name": "app-1",
- "lifetime": "3600",
- "launch_command": "sleep 3600",
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b"
- },
- "files": [
- {
- "type": "PROPERTIES",
- "dest_file": "file1",
- "props": {
- "k1": "v1",
- "k2": "v2"
- }
- },
- {
- "type": "XML",
- "dest_file": "file2",
- "props": {
- "k3": "v3"
- }
- }
- ]
- },
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "number_of_containers": 2,
- "components": [
- {
- "name": "simple",
- "configuration": {
- "files": [
- {
- "type": "PROPERTIES",
- "dest_file": "file1",
- "props": {
- "k1": "overridden"
- }
- }
- ]
- }
- },
- {
- "name": "master",
- "configuration": {
- "properties": {
- "name": "m",
- "g1": "overridden"
- }
- }
- },
- {
- "name": "worker",
- "resource": {
- "cpus": 1,
- "memory": "1024"
- },
- "configuration": {
- "properties": {
- "name": "worker",
- "g1": "overridden-by-worker",
- "timeout": "1000"
- }
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
deleted file mode 100644
index a163b33..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "name": "app-1",
- "lifetime": "3600",
- "launch_command": "sleep 3600",
- "configuration": {
- "properties": {
- "g1": "a",
- "g2": "b",
- "yarn.service.failure-count-reset.window": "60"
- }
- },
- "resource": {
- "cpus": 1,
- "memory": "512"
- },
- "number_of_containers": 2,
- "components": [
- {
- "name": "simple"
- },
- {
- "name": "master",
- "number_of_containers": 1,
- "configuration": {
- "properties": {
- "g1": "overridden",
- "g3": "will-be-overridden",
- "jvm.heapsize": "512M"
- }
- }
- },
- {
- "name": "worker",
- "number_of_containers": 5,
- "resource": {
- "cpus": 1,
- "memory": "1024"
- },
- "configuration": {
- "properties": {
- "g1": "overridden-by-worker",
- "jvm.heapsize": "512M"
- }
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
deleted file mode 100644
index 73d4e7b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "default-app-1",
- "lifetime": "3600",
- "components" :
- [
- {
- "name": "sleep",
- "number_of_containers": 1,
- "launch_command": "sleep 3600",
- "resource": {
- "cpus": 2,
- "memory": "256"
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
deleted file mode 100644
index 1f9dfeb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "name": "external-0",
- "lifetime": "3600",
- "artifact": {
- "type": "APPLICATION",
- "id": "app-1"
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
deleted file mode 100644
index 03ebce5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "name": "external-1",
- "lifetime": "3600",
- "components": [
- {
- "name": "simple",
- "artifact": {
- "type": "APPLICATION",
- "id": "app-1"
- }
- },
- {
- "name": "master",
- "configuration": {
- "properties": {
- "g3": "is-overridden"
- }
- }
- },
- {
- "name": "other",
- "launch_command": "sleep 3600",
- "number_of_containers": 2,
- "resource": {
- "cpus": 1,
- "memory": "512"
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
deleted file mode 100644
index 9e61fba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "name": "external-2",
- "lifetime": "3600",
- "components": [
- {
- "name": "ext",
- "artifact": {
- "type": "APPLICATION",
- "id": "external-1"
- }
- },
- {
- "name": "another",
- "launch_command": "sleep 3600",
- "number_of_containers": 1,
- "resource": {
- "cpus": 1,
- "memory": "512"
- }
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/appConfig.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/appConfig.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/appConfig.json
deleted file mode 100644
index c87f77c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/appConfig.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {},
- "global": {
- "am.config.generation": "true",
- "component.unique.names": "true",
-
- "export.app.monitor": "${COMPONENT1_HOST} : ${@//site/test-xml/xmlkey}",
- "export.other.key": "exportvalue",
-
- "docker.image": "docker.io/centos:centos6",
- "docker.startCommand": "sleep 600",
-
- "conf.test-json.type": "json",
- "conf.test-json.name": "/tmp/test.json",
- "conf.test-xml.type": "xml",
- "conf.test-xml.name": "/tmp/test.xml",
- "conf.test-properties.type": "properties",
- "conf.test-properties.name": "/tmp/test.xml",
- "conf.test-yaml.type": "yaml",
- "conf.test-yaml.name": "/tmp/test.yaml",
- "conf.test-env.type": "env",
- "conf.test-env.name": "/tmp/testenv",
- "conf.test-template.type": "template",
- "conf.test-template.name": "/tmp/test.template",
- "conf.test-hadoop-xml.type": "hadoop-xml",
- "conf.test-hadoop-xml.name": "/tmp/test-hadoop.xml",
-
- "site.test-json.jsonkey": "val1",
- "site.test-xml.xmlkey": "val2",
- "site.test-hadoop-xml.xmlkey": "val3",
- "site.test-properties.propkey": "val4",
- "site.test-yaml.yamlkey": "val5",
- "site.test-env.content": "test ${envkey1} {{envkey2}} content",
- "site.test-env.envkey1": "envval1",
- "site.test-env.envkey2": "envval2",
- "site.test-template.templatekey1": "templateval1",
- "site.test-template.templatekey2": "templateval2"
- },
- "components": {
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/resources.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/resources.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/resources.json
deleted file mode 100644
index 1b06224..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/resources.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {},
- "global": {},
- "components": {
- "slider-appmaster": {
- "yarn.memory": "384"
- },
- "COMPONENT": {
- "yarn.role.priority": "1",
- "yarn.component.instances": 2,
- "yarn.memory": "512",
- "yarn.vcores": "2"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/test.template
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/test.template b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/test.template
deleted file mode 100644
index 2922655..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/hadoop/yarn/service/provider/docker/test.template
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-test ${templatekey1} {{templatekey2}} content
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/yarn-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/yarn-site.xml
deleted file mode 100644
index 266caa9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/yarn-site.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-<configuration>
- <!-- Dummy (invalid) config file to be overwriten by TestYarnNativeServices with MiniCluster configuration. -->
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
deleted file mode 100644
index 6a208d8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>hadoop-yarn-applications</artifactId>
- <groupId>org.apache.hadoop</groupId>
- <version>3.0.0-beta1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider</artifactId>
- <name>Apache Hadoop YARN Slider</name>
- <packaging>pom</packaging>
-
- <properties>
- <hadoop.common.build.dir>${basedir}/../../../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
- </properties>
-
- <!-- Do not add dependencies here, add them to the POM of the leaf module -->
-
- <modules>
- <module>hadoop-yarn-slider-core</module>
- </modules>
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index 0fc5ceb..4fb579c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -36,7 +36,7 @@
<modules>
<module>hadoop-yarn-applications-distributedshell</module>
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
- <module>hadoop-yarn-slider</module>
+ <module>hadoop-yarn-services</module>
<module>hadoop-yarn-services-api</module>
</modules>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
index 4a048af..fa54a09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
@@ -1,3 +1,17 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
# YARN DNS Server
## Introduction
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
index e69de29..89fefe9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
@@ -0,0 +1,13 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 5440738..e4e611b 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -75,7 +75,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
- <excludePackageNames>org.apache.hadoop.yarn.proto:org.apache.hadoop.yarn.federation.proto:org.apache.slider</excludePackageNames>
+ <excludePackageNames>org.apache.hadoop.yarn.proto:org.apache.hadoop.yarn.federation.proto:org.apache.hadoop.yarn.service</excludePackageNames>
</configuration>
</plugin>
</plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
new file mode 100644
index 0000000..add2475
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
+
+/**
+ * Registry view for providers. This tracks where the service
+ * is registered, offers access to the record and other things.
+ */
+public class YarnRegistryViewForProviders {
+ private static final Log LOG =
+ LogFactory.getLog(YarnRegistryViewForProviders.class);
+
+ private final RegistryOperations registryOperations;
+ private final String user;
+ private final String sliderServiceClass;
+ private final String instanceName;
+ /**
+ * Record used where the service registered itself.
+ * Null until the service is registered
+ */
+ private ServiceRecord selfRegistration;
+
+ /**
+ * Path where record was registered
+ * Null until the service is registered
+ */
+ private String selfRegistrationPath;
+
+ public YarnRegistryViewForProviders(RegistryOperations registryOperations,
+ String user,
+ String sliderServiceClass,
+ String instanceName,
+ ApplicationAttemptId applicationAttemptId) {
+ Preconditions.checkArgument(registryOperations != null,
+ "null registry operations");
+ Preconditions.checkArgument(user != null, "null user");
+ Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
+ "unset service class");
+ Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+ "instanceName");
+ Preconditions.checkArgument(applicationAttemptId != null,
+ "null applicationAttemptId");
+ this.registryOperations = registryOperations;
+ this.user = user;
+ this.sliderServiceClass = sliderServiceClass;
+ this.instanceName = instanceName;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+
+ private void setSelfRegistration(ServiceRecord selfRegistration) {
+ this.selfRegistration = selfRegistration;
+ }
+
+ /**
+ * Get the path to where the service has registered itself.
+ * Null until the service is registered
+ * @return the service registration path.
+ */
+ public String getSelfRegistrationPath() {
+ return selfRegistrationPath;
+ }
+
+ /**
+ * Get the absolute path to where the service has registered itself.
+ * This includes the base registry path
+ * Null until the service is registered
+ * @return the service registration path.
+ */
+ public String getAbsoluteSelfRegistrationPath() {
+ if (selfRegistrationPath == null) {
+ return null;
+ }
+ String root = registryOperations.getConfig().getTrimmed(
+ RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
+ return RegistryPathUtils.join(root, selfRegistrationPath);
+ }
+
+ /**
+ * Add a component under the slider name/entry
+ * @param componentName component name
+ * @param record record to put
+ * @throws IOException
+ */
+ public void putComponent(String componentName,
+ ServiceRecord record) throws
+ IOException {
+ putComponent(sliderServiceClass, instanceName,
+ componentName,
+ record);
+ }
+
+ /**
+ * Add a component
+ * @param serviceClass service class to use under ~user
+ * @param componentName component name
+ * @param record record to put
+ * @throws IOException
+ */
+ public void putComponent(String serviceClass,
+ String serviceName,
+ String componentName,
+ ServiceRecord record) throws IOException {
+ String path = RegistryUtils.componentPath(
+ user, serviceClass, serviceName, componentName);
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ }
+
+ /**
+ * Add a service under a path, optionally purging any history
+ * @param username user
+ * @param serviceClass service class to use under ~user
+ * @param serviceName name of the service
+ * @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first.
+ * @return the path the service was created at
+ * @throws IOException
+ */
+ public String putService(String username,
+ String serviceClass,
+ String serviceName,
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ String path = RegistryUtils.servicePath(
+ username, serviceClass, serviceName);
+ if (deleteTreeFirst) {
+ registryOperations.delete(path, true);
+ }
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ return path;
+ }
+
+ /**
+ * Add a service under a path for the current user
+ * @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first
+ * @return the path the service was created at
+ * @throws IOException
+ */
+ public String registerSelf(
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ selfRegistrationPath =
+ putService(user, sliderServiceClass, instanceName, record, deleteTreeFirst);
+ setSelfRegistration(record);
+ return selfRegistrationPath;
+ }
+
+ /**
+ * Delete a component
+ * @param containerId component name
+ * @throws IOException
+ */
+ public void deleteComponent(ComponentInstanceId instanceId,
+ String containerId) throws IOException {
+ String path = RegistryUtils.componentPath(
+ user, sliderServiceClass, instanceName,
+ containerId);
+ LOG.info(instanceId + ": Deleting registry path " + path);
+ registryOperations.delete(path, false);
+ }
+
+ /**
+ * Delete the children of a path -but not the path itself.
+ * It is not an error if the path does not exist
+ * @param path path to delete
+ * @param recursive flag to request recursive deletes
+ * @throws IOException IO problems
+ */
+ public void deleteChildren(String path, boolean recursive) throws IOException {
+ List<String> childNames = null;
+ try {
+ childNames = registryOperations.list(path);
+ } catch (PathNotFoundException e) {
+ return;
+ }
+ for (String childName : childNames) {
+ String child = join(path, childName);
+ registryOperations.delete(child, recursive);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
new file mode 100644
index 0000000..cf4e836
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.commons.configuration2.SubsetConfiguration;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsSink;
+import org.apache.hadoop.metrics2.MetricsTag;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Write the metrics to a ATSv2. Generally, this class is instantiated via
+ * hadoop-metrics2 property files. Specifically, you would create this class by
+ * adding the following to by This would actually be set as: <code>
+ * [prefix].sink.[some instance name].class
+ * =org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink
+ * </code>, where <tt>prefix</tt> is "atsv2": and <tt>some instance name</tt> is
+ * just any unique name, so properties can be differentiated if there are
+ * multiple sinks of the same type created
+ */
+public class ServiceMetricsSink implements MetricsSink {
+
+ private static final Logger log =
+ LoggerFactory.getLogger(ServiceMetricsSink.class);
+
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+
+ public ServiceMetricsSink() {
+
+ }
+
+ public ServiceMetricsSink(ServiceTimelinePublisher publisher) {
+ serviceTimelinePublisher = publisher;
+ }
+
+ /**
+ * Publishes service and component metrics to ATS.
+ */
+ @Override
+ public void putMetrics(MetricsRecord record) {
+ if (serviceTimelinePublisher.isStopped()) {
+ log.warn("ServiceTimelinePublisher has stopped. "
+ + "Not publishing any more metrics to ATS.");
+ return;
+ }
+
+ boolean isServiceMetrics = false;
+ boolean isComponentMetrics = false;
+ String appId = null;
+ for (MetricsTag tag : record.tags()) {
+ if (tag.name().equals("type") && tag.value().equals("service")) {
+ isServiceMetrics = true;
+ } else if (tag.name().equals("type") && tag.value().equals("component")) {
+ isComponentMetrics = true;
+ break; // if component metrics, no more information required from tag so
+ // break the loop
+ } else if (tag.name().equals("appId")) {
+ appId = tag.value();
+ }
+ }
+
+ if (isServiceMetrics && appId != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("Publishing service metrics. " + record);
+ }
+ serviceTimelinePublisher.publishMetrics(record.metrics(), appId,
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString(),
+ record.timestamp());
+ } else if (isComponentMetrics) {
+ if (log.isDebugEnabled()) {
+ log.debug("Publishing Component metrics. " + record);
+ }
+ serviceTimelinePublisher.publishMetrics(record.metrics(), record.name(),
+ ServiceTimelineEntityType.COMPONENT.toString(), record.timestamp());
+ }
+ }
+
+ @Override
+ public void init(SubsetConfiguration conf) {
+ }
+
+ @Override
+ public void flush() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
new file mode 100644
index 0000000..d5c9539
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+/**
+ * Slider entities that are published to ATS.
+ */
+public enum ServiceTimelineEntityType {
+ /**
+ * Used for publishing service entity information.
+ */
+ SERVICE_ATTEMPT,
+
+ /**
+ * Used for publishing component entity information.
+ */
+ COMPONENT,
+
+ /**
+ * Used for publishing component instance entity information.
+ */
+ COMPONENT_INSTANCE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
new file mode 100644
index 0000000..7f7f9a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+/**
+ * Events that are used to store in ATS.
+ */
+public enum ServiceTimelineEvent {
+ SERVICE_ATTEMPT_REGISTERED,
+
+ SERVICE_ATTEMPT_UNREGISTERED,
+
+ COMPONENT_INSTANCE_REGISTERED,
+
+ COMPONENT_INSTANCE_UNREGISTERED,
+
+ COMPONENT_INSTANCE_UPDATED
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
new file mode 100644
index 0000000..78a7171
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+/**
+ * Constants which are stored as key in ATS
+ */
+public final class ServiceTimelineMetricsConstants {
+
+ public static final String URI = "URI";
+
+ public static final String NAME = "NAME";
+
+ public static final String STATE = "STATE";
+
+ public static final String EXIT_STATUS_CODE = "EXIT_STATUS_CODE";
+
+ public static final String EXIT_REASON = "EXIT_REASON";
+
+ public static final String DIAGNOSTICS_INFO = "DIAGNOSTICS_INFO";
+
+ public static final String LAUNCH_TIME = "LAUNCH_TIME";
+
+ public static final String QUICK_LINKS = "QUICK_LINKS";
+
+ public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND";
+
+ public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS";
+
+ public static final String RUNNING_CONTAINERS =
+ "NUMBER_OF_RUNNING_CONTAINERS";
+
+ /**
+ * Artifacts constants.
+ */
+ public static final String ARTIFACT_ID = "ARTIFACT_ID";
+
+ public static final String ARTIFACT_TYPE = "ARTIFACT_TYPE";
+
+ public static final String ARTIFACT_URI = "ARTIFACT_URI";
+
+ /**
+ * Resource constants.
+ */
+ public static final String RESOURCE_CPU = "RESOURCE_CPU";
+
+ public static final String RESOURCE_MEMORY = "RESOURCE_MEMORY";
+
+ public static final String RESOURCE_PROFILE = "RESOURCE_PROFILE";
+
+ /**
+ * component instance constants.
+ */
+ public static final String IP = "IP";
+
+ public static final String HOSTNAME = "HOSTNAME";
+
+ public static final String BARE_HOST = "BARE_HOST";
+
+ public static final String COMPONENT_NAME = "COMPONENT_NAME";
+
+ public static final String COMPONENT_INSTANCE_NAME = "COMPONENT_INSTANCE_NAME";
+
+ /**
+ * component constants.
+ */
+ public static final String DEPENDENCIES = "DEPENDENCIES";
+
+ public static final String DESCRIPTION = "DESCRIPTION";
+
+ public static final String RUN_PRIVILEGED_CONTAINER =
+ "RUN_PRIVILEGED_CONTAINER";
+
+ public static final String PLACEMENT_POLICY = "PLACEMENT_POLICY";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
new file mode 100644
index 0000000..5e65ad9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.service.timelineservice.ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO;
+
+/**
+ * A single service that publishes all the Timeline Entities.
+ */
+public class ServiceTimelinePublisher extends CompositeService {
+
+ // Number of bytes of config which can be published in one shot to ATSv2.
+ public static final int ATS_CONFIG_PUBLISH_SIZE_BYTES = 10 * 1024;
+
+ private TimelineV2Client timelineClient;
+
+ private volatile boolean stopped = false;
+
+ private static final Logger log =
+ LoggerFactory.getLogger(ServiceTimelinePublisher.class);
+
+ @Override
+ protected void serviceInit(org.apache.hadoop.conf.Configuration configuration)
+ throws Exception {
+ addService(timelineClient);
+ }
+
+
+ @Override
+ protected void serviceStop() throws Exception {
+ stopped = true;
+ super.serviceStop();
+ }
+
+ public boolean isStopped() {
+ return stopped;
+ }
+
+ public ServiceTimelinePublisher(TimelineV2Client client) {
+ super(ServiceTimelinePublisher.class.getName());
+ timelineClient = client;
+ }
+
+ public void serviceAttemptRegistered(Service service,
+ org.apache.hadoop.conf.Configuration systemConf) {
+ long currentTimeMillis = service.getLaunchTime() == null
+ ? System.currentTimeMillis() : service.getLaunchTime().getTime();
+
+ TimelineEntity entity = createServiceAttemptEntity(service.getId());
+ entity.setCreatedTime(currentTimeMillis);
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName());
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ service.getState().toString());
+ entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
+ currentTimeMillis);
+ entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
+ service.getQuicklinks());
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent.setId(ServiceTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString());
+ startEvent.setTimestamp(currentTimeMillis);
+ entity.addEvent(startEvent);
+
+ // publish before configurations published
+ putEntity(entity);
+
+ // publish system config - YarnConfiguration
+ populateTimelineEntity(systemConf.iterator(), service.getId(),
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
+ // publish user conf
+ publishUserConf(service.getConfiguration(), service.getId(),
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
+
+ // publish component as separate entity.
+ publishComponents(service.getComponents());
+ }
+
+ public void serviceAttemptUpdated(Service service) {
+ TimelineEntity entity = createServiceAttemptEntity(service.getId());
+ entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
+ service.getQuicklinks());
+ putEntity(entity);
+ }
+
+ public void serviceAttemptUnregistered(ServiceContext context,
+ String diagnostics) {
+ TimelineEntity entity = createServiceAttemptEntity(
+ context.attemptId.getApplicationId().toString());
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ FinalApplicationStatus.ENDED);
+ entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent finishEvent = new TimelineEvent();
+ finishEvent
+ .setId(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString());
+ finishEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(finishEvent);
+
+ putEntity(entity);
+ }
+
+ public void componentInstanceStarted(Container container,
+ ComponentInstance instance) {
+
+ TimelineEntity entity = createComponentInstanceEntity(container.getId());
+ entity.setCreatedTime(container.getLaunchTime().getTime());
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.BARE_HOST,
+ container.getBareHost());
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ container.getState().toString());
+ entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
+ container.getLaunchTime().getTime());
+ entityInfos.put(ServiceTimelineMetricsConstants.COMPONENT_NAME,
+ instance.getCompName());
+ entityInfos.put(ServiceTimelineMetricsConstants.COMPONENT_INSTANCE_NAME,
+ instance.getCompInstanceName());
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent
+ .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_REGISTERED.toString());
+ startEvent.setTimestamp(container.getLaunchTime().getTime());
+ entity.addEvent(startEvent);
+
+ putEntity(entity);
+ }
+
+ public void componentInstanceFinished(ComponentInstance instance,
+ int exitCode, ContainerState state, String diagnostics) {
+ TimelineEntity entity = createComponentInstanceEntity(
+ instance.getContainer().getId().toString());
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE,
+ exitCode);
+ entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE, state);
+ entity.addInfo(entityInfos);
+
+ // add an event
+ TimelineEvent startEvent = new TimelineEvent();
+ startEvent
+ .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UNREGISTERED.toString());
+ startEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(startEvent);
+
+ putEntity(entity);
+ }
+
+ public void componentInstanceUpdated(Container container) {
+ TimelineEntity entity = createComponentInstanceEntity(container.getId());
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.IP, container.getIp());
+ entityInfos.put(ServiceTimelineMetricsConstants.HOSTNAME,
+ container.getHostname());
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ container.getState().toString());
+ entity.addInfo(entityInfos);
+
+ TimelineEvent updateEvent = new TimelineEvent();
+ updateEvent
+ .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UPDATED.toString());
+ updateEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(updateEvent);
+
+ putEntity(entity);
+ }
+
+ private void publishComponents(List<Component> components) {
+ long currentTimeMillis = System.currentTimeMillis();
+ for (Component component : components) {
+ TimelineEntity entity = createComponentEntity(component.getName());
+ entity.setCreatedTime(currentTimeMillis);
+
+ // create info keys
+ Map<String, Object> entityInfos = new HashMap<String, Object>();
+ entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_ID,
+ component.getArtifact().getId());
+ entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_TYPE,
+ component.getArtifact().getType().toString());
+ if (component.getResource().getProfile() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_PROFILE,
+ component.getResource().getProfile());
+ }
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_CPU,
+ component.getResource().getCpus());
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_MEMORY,
+ component.getResource().getMemory());
+
+ if (component.getLaunchCommand() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_COMMAND,
+ component.getLaunchCommand());
+ }
+ entityInfos.put(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER,
+ component.getRunPrivilegedContainer().toString());
+ if (component.getPlacementPolicy() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.PLACEMENT_POLICY,
+ component.getPlacementPolicy().getLabel());
+ }
+ entity.addInfo(entityInfos);
+
+ putEntity(entity);
+
+ // publish component specific configurations
+ publishUserConf(component.getConfiguration(), component.getName(),
+ ServiceTimelineEntityType.COMPONENT.toString());
+ }
+ }
+
+ private void publishUserConf(Configuration configuration,
+ String entityId, String entityType) {
+ populateTimelineEntity(configuration.getProperties().entrySet().iterator(),
+ entityId, entityType);
+
+ populateTimelineEntity(configuration.getEnv().entrySet().iterator(),
+ entityId, entityType);
+
+ for (ConfigFile configFile : configuration.getFiles()) {
+ populateTimelineEntity(configFile.getProps().entrySet().iterator(),
+ entityId, entityType);
+ }
+ }
+
+ private void populateTimelineEntity(Iterator<Entry<String, String>> iterator,
+ String entityId, String entityType) {
+ int configSize = 0;
+ TimelineEntity entity = createTimelineEntity(entityId, entityType);
+ while (iterator.hasNext()) {
+ Entry<String, String> entry = iterator.next();
+ int size = entry.getKey().length() + entry.getValue().length();
+ configSize += size;
+ // Configs are split into multiple entities if they exceed 100kb in size.
+ if (configSize > ATS_CONFIG_PUBLISH_SIZE_BYTES) {
+ if (entity.getConfigs().size() > 0) {
+ putEntity(entity);
+ entity = createTimelineEntity(entityId, entityType);
+ }
+ configSize = size;
+ }
+ entity.addConfig(entry.getKey(), entry.getValue());
+ }
+ if (configSize > 0) {
+ putEntity(entity);
+ }
+ }
+
+ /**
+ * Called from ServiceMetricsSink at regular interval of time.
+ * @param metrics of service or components
+ * @param entityId Id of entity
+ * @param entityType Type of entity
+ * @param timestamp
+ */
+ public void publishMetrics(Iterable<AbstractMetric> metrics, String entityId,
+ String entityType, long timestamp) {
+ TimelineEntity entity = createTimelineEntity(entityId, entityType);
+ Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
+ for (AbstractMetric metric : metrics) {
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setId(metric.name());
+ timelineMetric.addValue(timestamp, metric.value());
+ entityMetrics.add(timelineMetric);
+ }
+ entity.setMetrics(entityMetrics);
+ putEntity(entity);
+ }
+
+ private TimelineEntity createServiceAttemptEntity(String serviceId) {
+ TimelineEntity entity = createTimelineEntity(serviceId,
+ ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
+ return entity;
+ }
+
+ private TimelineEntity createComponentInstanceEntity(String instanceId) {
+ TimelineEntity entity = createTimelineEntity(instanceId,
+ ServiceTimelineEntityType.COMPONENT_INSTANCE.toString());
+ return entity;
+ }
+
+ private TimelineEntity createComponentEntity(String componentId) {
+ TimelineEntity entity = createTimelineEntity(componentId,
+ ServiceTimelineEntityType.COMPONENT.toString());
+ return entity;
+ }
+
+ private TimelineEntity createTimelineEntity(String entityId,
+ String entityType) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(entityId);
+ entity.setType(entityType);
+ return entity;
+ }
+
+ private void putEntity(TimelineEntity entity) {
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug("Publishing the entity " + entity + ", JSON-style content: "
+ + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ }
+ if (timelineClient != null) {
+ timelineClient.putEntitiesAsync(entity);
+ } else {
+ log.error("Seems like client has been removed before the entity "
+ + "could be published for " + entity);
+ }
+ } catch (Exception e) {
+ log.error("Error when publishing entity " + entity, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
new file mode 100644
index 0000000..72f7842
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * ATS implementation
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
new file mode 100644
index 0000000..2607c08
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+
+import java.io.IOException;
+
+/**
+ * Persistence of {@link SerializedApplicationReport}
+ *
+ */
+public class ApplicationReportSerDeser
+ extends JsonSerDeser<SerializedApplicationReport> {
+ public ApplicationReportSerDeser() {
+ super(SerializedApplicationReport.class);
+ }
+
+
+ private static final ApplicationReportSerDeser
+ staticinstance = new ApplicationReportSerDeser();
+
+ /**
+ * Convert an instance to a JSON string -sync access to a shared ser/deser
+ * object instance
+ * @param instance object to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public static String toString(SerializedApplicationReport instance)
+ throws IOException, JsonGenerationException, JsonMappingException {
+ synchronized (staticinstance) {
+ return staticinstance.toJson(instance);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
new file mode 100644
index 0000000..86896b2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.encodeForRegistry;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.convertUsername;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.getCurrentUsernameUnencoded;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
+
+/**
+ * Generic code to get the URLs for clients via the registry
+ */
+public class ClientRegistryBinder {
+ private static final Logger log =
+ LoggerFactory.getLogger(ClientRegistryBinder.class);
+
+ private final RegistryOperations operations;
+
+ public ClientRegistryBinder(RegistryOperations operations) {
+ this.operations = operations;
+ }
+
+ /**
+ * Buld the user path -switches to the system path if the user is "".
+ * It also cross-converts the username to ascii via punycode
+ * @param username username or ""
+ * @return the path to the user
+ */
+ public static String homePathForUser(String username) {
+ Preconditions.checkArgument(username != null, "null user");
+
+ // catch recursion
+ if (username.startsWith(RegistryConstants.PATH_USERS)) {
+ return username;
+ }
+
+ if (username.isEmpty()) {
+ return RegistryConstants.PATH_SYSTEM_SERVICES;
+ }
+
+ // convert username to registry name
+ String convertedName = convertUsername(username);
+
+ return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
+ encodeForRegistry(convertedName));
+ }
+
+ /**
+ * Get the current username, before any encoding has been applied.
+ * @return the current user from the kerberos identity, falling back
+ * to the user and/or env variables.
+ */
+ public static String currentUsernameUnencoded() {
+ String env_hadoop_username = System.getenv(
+ RegistryInternalConstants.HADOOP_USER_NAME);
+ return getCurrentUsernameUnencoded(env_hadoop_username);
+ }
+
+ /**
+ * Qualify a user.
+ * <ol>
+ * <li> <code>"~"</code> maps to user home path home</li>
+ * <li> <code>"~user"</code> maps to <code>/users/$user</code></li>
+ * <li> <code>"/"</code> maps to <code>/services/</code></li>
+ * </ol>
+ * @param user the username
+ * @return the base path
+ */
+ public static String qualifyUser(String user) {
+ // qualify the user
+ String t = user.trim();
+ if (t.startsWith("/")) {
+ // already resolved
+ return t;
+ } else if (t.equals("~")) {
+ // self
+ return currentUsernameUnencoded();
+ } else if (t.startsWith("~")) {
+ // another user
+ // convert username to registry name
+ String convertedName = convertUsername(t.substring(1));
+
+ return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
+ encodeForRegistry(convertedName));
+ } else {
+ return "/" + t;
+ }
+ }
+
+ /**
+ * Look up an external REST API
+ * @param user user which will be qualified as per {@link #qualifyUser(String)}
+ * @param serviceClass service class
+ * @param instance instance name
+ * @param api API
+ * @return the API, or an exception is raised.
+ * @throws IOException
+ */
+ public String lookupExternalRestAPI(String user,
+ String serviceClass,
+ String instance,
+ String api)
+ throws IOException {
+ String qualified = qualifyUser(user);
+ String path = servicePath(qualified, serviceClass, instance);
+ String restAPI = resolveExternalRestAPI(api, path);
+ if (restAPI == null) {
+ throw new PathNotFoundException(path + " API " + api);
+ }
+ return restAPI;
+ }
+
+ /**
+ * Resolve a service record then return an external REST API exported it.
+ *
+ * @param api API to resolve
+ * @param path path of the service record
+ * @return null if the record exists but the API is absent or it has no
+ * REST endpoints.
+ * @throws IOException resolution problems, as covered in
+ * {@link RegistryOperations#resolve(String)}
+ */
+ protected String resolveExternalRestAPI(String api, String path) throws
+ IOException {
+ ServiceRecord record = operations.resolve(path);
+ return lookupRestAPI(record, api, true);
+ }
+
+ /**
+ * Look up an external REST API endpoint
+ * @param record service record
+ * @param api URI of api
+ * @param external flag to indicate this is an external record
+ * @return the first endpoint of the implementation, or null if there
+ * is no entry for the API, implementation or it's the wrong type.
+ */
+ public static String lookupRestAPI(ServiceRecord record,
+ String api, boolean external) throws InvalidRecordException {
+ try {
+ String url = null;
+ Endpoint endpoint = getEndpoint(record, api, external);
+ List<String> addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(endpoint);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ return url;
+ } catch (InvalidRecordException e) {
+ log.debug("looking for API {}", api, e);
+ return null;
+ }
+ }
+
+ /**
+ * Get an endpont by API
+ * @param record service record
+ * @param api API
+ * @param external flag to indicate this is an external record
+ * @return the endpoint or null
+ */
+ public static Endpoint getEndpoint(ServiceRecord record,
+ String api,
+ boolean external) {
+ return external ? record.getExternalEndpoint(api)
+ : record.getInternalEndpoint(api);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
new file mode 100644
index 0000000..9f0e5d4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Some general comparators
+ */
+public class Comparators {
+
+ public static class LongComparator implements Comparator<Long>, Serializable {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return o1.compareTo(o2);
+ }
+ }
+
+ public static class InvertedLongComparator
+ implements Comparator<Long>, Serializable {
+ @Override
+ public int compare(Long o1, Long o2) {
+ return o2.compareTo(o1);
+ }
+ }
+
+ /**
+ * Little template class to reverse any comparitor
+ * @param <CompareType> the type that is being compared
+ */
+ public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
+ Serializable {
+
+ final Comparator<CompareType> instance;
+
+ public ComparatorReverser(Comparator<CompareType> instance) {
+ this.instance = instance;
+ }
+
+ @Override
+ public int compare(CompareType first, CompareType second) {
+ return instance.compare(second, first);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
new file mode 100644
index 0000000..fe8cce8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URL;
+import java.util.Map;
+
+/**
+ * Methods to aid in config, both in the Configuration class and
+ * with other parts of setting up Slider-initated processes.
+ *
+ * Some of the methods take an argument of a map iterable for their sources; this allows
+ * the same method
+ */
+public class ConfigHelper {
+ private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Map<String, String> map,
+ String origin) throws BadConfigException {
+ addConfigMap(config, map.entrySet(), origin);
+ }
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Iterable<Map.Entry<String, String>> map,
+ String origin) throws BadConfigException {
+ for (Map.Entry<String, String> mapEntry : map) {
+ String key = mapEntry.getKey();
+ String value = mapEntry.getValue();
+ if (value == null) {
+ throw new BadConfigException("Null value for property " + key);
+ }
+ config.set(key, value, origin);
+ }
+ }
+
+ /**
+ * Convert to an XML string
+ * @param conf configuration
+ * @return conf
+ * @throws IOException
+ */
+ public static String toXml(Configuration conf) throws IOException {
+ StringWriter writer = new StringWriter();
+ conf.writeXml(writer);
+ return writer.toString();
+ }
+
+
+ /**
+ * Register a resource as a default resource.
+ * Do not attempt to use this unless you understand that the
+ * order in which default resources are loaded affects the outcome,
+ * and that subclasses of Configuration often register new default
+ * resources
+ * @param resource the resource name
+ * @return the URL or null
+ */
+ public static URL registerDefaultResource(String resource) {
+ URL resURL = getResourceUrl(resource);
+ if (resURL != null) {
+ Configuration.addDefaultResource(resource);
+ }
+ return resURL;
+ }
+
+ /**
+ * Load a configuration from a resource on this classpath.
+ * If the resource is not found, an empty configuration is returned
+ * @param resource the resource name
+ * @return the loaded configuration.
+ */
+ public static Configuration loadFromResource(String resource) {
+ Configuration conf = new Configuration(false);
+ URL resURL = getResourceUrl(resource);
+ if (resURL != null) {
+ log.debug("loaded resources from {}", resURL);
+ conf.addResource(resource);
+ } else{
+ log.debug("failed to find {} on the classpath", resource);
+ }
+ return conf;
+
+ }
+
+ /**
+ * Get the URL to a resource, null if not on the CP
+ * @param resource resource to look for
+ * @return the URL or null
+ */
+ public static URL getResourceUrl(String resource) {
+ return ConfigHelper.class.getClassLoader()
+ .getResource(resource);
+ }
+
+ /**
+ * This goes through the keyset of one configuration and retrieves each value
+ * from a value source -a different or the same configuration. This triggers
+ * the property resolution process of the value, resolving any variables against
+ * in-config or inherited configurations
+ * @param keysource source of keys
+ * @param valuesource the source of values
+ * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
+ */
+ public static Configuration resolveConfiguration(
+ Iterable<Map.Entry<String, String>> keysource,
+ Configuration valuesource) {
+ Configuration result = new Configuration(false);
+ for (Map.Entry<String, String> entry : keysource) {
+ String key = entry.getKey();
+ String value = valuesource.get(key);
+ Preconditions.checkState(value != null,
+ "no reference for \"%s\" in values", key);
+ result.set(key, value);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
new file mode 100644
index 0000000..a969be9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ConfigUtils {
+ public static final String TEMPLATE_FILE = "template.file";
+
+ public static String replaceProps(Map<String, String> config, String content) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ tokens.put("{{" + entry.getKey() + "}}", entry.getValue());
+ }
+ String value = content;
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ return value;
+ }
+
+ public static Map<String, String> replacePropsInConfig(
+ Map<String, String> config, Map<String, String> env) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : env.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ }
+ Map<String, String> newConfig = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ String value = entry.getValue();
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ newConfig.put(entry.getKey(), entry.getValue());
+ }
+ return newConfig;
+ }
+
+ public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
+ Map<String, String> config, SliderFileSystem fileSystem,
+ String clusterName, String fileName) throws IOException {
+ if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
+ return;
+ }
+ Path templateFile = null;
+ if (config.containsKey(TEMPLATE_FILE)) {
+ templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ config.get(TEMPLATE_FILE));
+ }
+ if (!fileSystem.isFile(templateFile)) {
+ throw new IOException("config specified template file " + config
+ .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
+ }
+ }
+ if (templateFile == null && fileName != null) {
+ templateFile = fileSystem.buildResourcePath(fileName);
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ fileName);
+ }
+ }
+ if (fileSystem.isFile(templateFile)) {
+ config.put("content", fileSystem.cat(templateFile));
+ } else {
+ config.put("content", "");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
new file mode 100644
index 0000000..281e1dfe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -0,0 +1,521 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.util.Records;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CoreFileSystem {
+ private static final Logger
+ log = LoggerFactory.getLogger(CoreFileSystem.class);
+
+ private static final String UTF_8 = "UTF-8";
+
+ protected final FileSystem fileSystem;
+ protected final Configuration configuration;
+
+ public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
+ Preconditions.checkNotNull(fileSystem,
+ "Cannot create a CoreFileSystem with a null FileSystem");
+ Preconditions.checkNotNull(configuration,
+ "Cannot create a CoreFileSystem with a null Configuration");
+ this.fileSystem = fileSystem;
+ this.configuration = configuration;
+ }
+
+ public CoreFileSystem(Configuration configuration) throws IOException {
+ Preconditions.checkNotNull(configuration,
+ "Cannot create a CoreFileSystem with a null Configuration");
+ this.fileSystem = FileSystem.get(configuration);
+ this.configuration = configuration;
+ }
+
+ /**
+ * Get the temp path for this cluster
+ * @param clustername name of the cluster
+ * @return path for temp files (is not purged)
+ */
+ public Path getTempPathForCluster(String clustername) {
+ Path clusterDir = buildClusterDirPath(clustername);
+ return new Path(clusterDir, YarnServiceConstants.TMP_DIR_PREFIX);
+ }
+
+ /**
+ * Returns the underlying FileSystem for this object.
+ *
+ * @return filesystem
+ */
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("CoreFileSystem{");
+ sb.append("fileSystem=").append(fileSystem.getUri());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Build up the path string for a cluster instance -no attempt to
+ * create the directory is made
+ *
+ * @param clustername name of the cluster
+ * @return the path for persistent data
+ */
+ public Path buildClusterDirPath(String clustername) {
+ Preconditions.checkNotNull(clustername);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.SERVICES_DIRECTORY + "/" + clustername);
+ }
+
+
+ /**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab
+ */
+ public Path buildKeytabInstallationDirPath(String keytabFolder) {
+ Preconditions.checkNotNull(keytabFolder);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.KEYTAB_DIR + "/" + keytabFolder);
+ }
+
+ /**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab installation location
+ */
+ public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
+ Path homePath = getHomeDirectory();
+ Path baseKeytabDir;
+ if (keytabDir != null) {
+ baseKeytabDir = new Path(homePath, keytabDir);
+ } else {
+ baseKeytabDir = new Path(buildClusterDirPath(clusterName),
+ YarnServiceConstants.KEYTAB_DIR);
+ }
+ return keytabName == null ? baseKeytabDir :
+ new Path(baseKeytabDir, keytabName);
+ }
+
+ /**
+ * Build up the path string for resource install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildResourcePath(String resourceFolder) {
+ Preconditions.checkNotNull(resourceFolder);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + resourceFolder);
+ }
+
+ /**
+ * Build up the path string for resource install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for resource
+ */
+ public Path buildResourcePath(String dirName, String fileName) {
+ Preconditions.checkNotNull(dirName);
+ Preconditions.checkNotNull(fileName);
+ Path path = getBaseApplicationPath();
+ return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + dirName + "/" + fileName);
+ }
+
+ /**
+ * Create a directory with the given permissions.
+ *
+ * @param dir directory
+ * @param clusterPerms cluster permissions
+ * @throws IOException IO problem
+ * @throws BadClusterStateException any cluster state problem
+ */
+ @SuppressWarnings("deprecation")
+ public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
+ IOException,
+ BadClusterStateException {
+ if (fileSystem.isFile(dir)) {
+ // HADOOP-9361 shows some filesystems don't correctly fail here
+ throw new BadClusterStateException(
+ "Cannot create a directory over a file %s", dir);
+ }
+ log.debug("mkdir {} with perms {}", dir, clusterPerms);
+ //no mask whatoever
+ fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
+ fileSystem.mkdirs(dir, clusterPerms);
+ //and force set it anyway just to make sure
+ fileSystem.setPermission(dir, clusterPerms);
+ }
+
+ /**
+ * Verify that the cluster directory is not present
+ *
+ * @param clustername name of the cluster
+ * @param clusterDirectory actual directory to look for
+ * @throws IOException trouble with FS
+ * @throws SliderException If the directory exists
+ */
+ public void verifyClusterDirectoryNonexistent(String clustername,
+ Path clusterDirectory)
+ throws IOException, SliderException {
+ if (fileSystem.exists(clusterDirectory)) {
+ throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
+ clusterDirectory);
+ }
+ }
+ /**
+ * Verify that the given directory is not present
+ *
+ * @param clusterDirectory actual directory to look for
+ * @throws IOException trouble with FS
+ * @throws SliderException If the directory exists
+ */
+ public void verifyDirectoryNonexistent(Path clusterDirectory) throws
+ IOException,
+ SliderException {
+ if (fileSystem.exists(clusterDirectory)) {
+
+ log.error("Dir {} exists: {}",
+ clusterDirectory,
+ listFSDir(clusterDirectory));
+ throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
+ clusterDirectory);
+ }
+ }
+
+ /**
+ * Verify that a user has write access to a directory.
+ * It does this by creating then deleting a temp file
+ *
+ * @param dirPath actual directory to look for
+ * @throws FileNotFoundException file not found
+ * @throws IOException trouble with FS
+ * @throws BadClusterStateException if the directory is not writeable
+ */
+ public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
+ SliderException {
+ verifyPathExists(dirPath);
+ Path tempFile = new Path(dirPath, "tmp-file-for-checks");
+ try {
+ FSDataOutputStream out ;
+ out = fileSystem.create(tempFile, true);
+ IOUtils.closeStream(out);
+ fileSystem.delete(tempFile, false);
+ } catch (IOException e) {
+ log.warn("Failed to create file {}: {}", tempFile, e);
+ throw new BadClusterStateException(e,
+ "Unable to write to directory %s : %s", dirPath, e.toString());
+ }
+ }
+
+ /**
+ * Verify that a path exists
+ * @param path path to check
+ * @throws FileNotFoundException file not found
+ * @throws IOException trouble with FS
+ */
+ public void verifyPathExists(Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ throw new FileNotFoundException(path.toString());
+ }
+ }
+
+ /**
+ * Verify that a path exists
+ * @param path path to check
+ * @throws FileNotFoundException file not found or is not a file
+ * @throws IOException trouble with FS
+ */
+ public void verifyFileExists(Path path) throws IOException {
+ FileStatus status = fileSystem.getFileStatus(path);
+
+ if (!status.isFile()) {
+ throw new FileNotFoundException("Not a file: " + path.toString());
+ }
+ }
+
+ /**
+ * Given a path, check if it exists and is a file
+ *
+ * @param path
+ * absolute path to the file to check
+ * @return true if and only if path exists and is a file, false for all other
+ * reasons including if file check throws IOException
+ */
+ public boolean isFile(Path path) {
+ boolean isFile = false;
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ if (status.isFile()) {
+ isFile = true;
+ }
+ } catch (IOException e) {
+ // ignore, isFile is already set to false
+ }
+ return isFile;
+ }
+
+ /**
+ * Get the base path
+ *
+ * @return the base path optionally configured by
+ * {@link YarnServiceConf#YARN_SERVICE_BASE_PATH}
+ */
+ public Path getBaseApplicationPath() {
+ String configuredBasePath = configuration
+ .get(YarnServiceConf.YARN_SERVICE_BASE_PATH,
+ getHomeDirectory() + "/" + YarnServiceConstants.SERVICE_BASE_DIRECTORY);
+ return new Path(configuredBasePath);
+ }
+
+ /**
+ * Get slider dependency parent dir in HDFS
+ *
+ * @return the parent dir path of slider.tar.gz in HDFS
+ */
+ public Path getDependencyPath() {
+ String parentDir = YarnServiceConstants.DEPENDENCY_DIR;
+ return new Path(String.format(parentDir, VersionInfo.getVersion()));
+ }
+
+ /**
+ * Get slider.tar.gz absolute filepath in HDFS
+ *
+ * @return the absolute path to slider.tar.gz in HDFS
+ */
+ public Path getDependencyTarGzip() {
+ Path dependencyLibAmPath = getDependencyPath();
+ Path dependencyLibTarGzip = new Path(
+ dependencyLibAmPath.toUri().toString(),
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+ + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ return dependencyLibTarGzip;
+ }
+
+ public Path getHomeDirectory() {
+ return fileSystem.getHomeDirectory();
+ }
+
+ /**
+ * Create an AM resource from the
+ *
+ * @param destPath dest path in filesystem
+ * @param resourceType resource type
+ * @return the local resource for AM
+ */
+ public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
+ FileStatus destStatus = fileSystem.getFileStatus(destPath);
+ LocalResource amResource = Records.newRecord(LocalResource.class);
+ amResource.setType(resourceType);
+ // Set visibility of the resource
+ // Setting to most private option
+ amResource.setVisibility(LocalResourceVisibility.APPLICATION);
+ // Set the resource to be copied over
+ amResource.setResource(
+ URL.fromPath(fileSystem.resolvePath(destStatus.getPath())));
+ // Set timestamp and length of file so that the framework
+ // can do basic sanity checks for the local resource
+ // after it has been copied over to ensure it is the same
+ // resource the client intended to use with the service
+ amResource.setTimestamp(destStatus.getModificationTime());
+ amResource.setSize(destStatus.getLen());
+ return amResource;
+ }
+
+ /**
+ * Register all files under a fs path as a directory to push out
+ *
+ * @param srcDir src dir
+ * @param destRelativeDir dest dir (no trailing /)
+ * @return the map of entries
+ */
+ public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
+ //now register each of the files in the directory to be
+ //copied to the destination
+ FileStatus[] fileset = fileSystem.listStatus(srcDir);
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>(fileset.length);
+ for (FileStatus entry : fileset) {
+
+ LocalResource resource = createAmResource(entry.getPath(),
+ LocalResourceType.FILE);
+ String relativePath = destRelativeDir + "/" + entry.getPath().getName();
+ localResources.put(relativePath, resource);
+ }
+ return localResources;
+ }
+
+ /**
+ * Submit a JAR containing a specific class, returning
+ * the resource to be mapped in
+ *
+ * @param clazz class to look for
+ * @param subdir subdirectory (expected to end in a "/")
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
+ throws IOException, SliderException {
+ File localFile = SliderUtils.findContainingJarOrFail(clazz);
+ return submitFile(localFile, tempPath, subdir, jarName);
+ }
+
+ /**
+ * Submit a local file to the filesystem references by the instance's cluster
+ * filesystem
+ *
+ * @param localFile filename
+ * @param subdir subdirectory (expected to end in a "/")
+ * @param destFileName destination filename
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName)
+ throws IOException {
+ Path src = new Path(localFile.toString());
+ Path subdirPath = new Path(tempPath, subdir);
+ fileSystem.mkdirs(subdirPath);
+ Path destPath = new Path(subdirPath, destFileName);
+ log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath);
+
+ fileSystem.copyFromLocalFile(false, true, src, destPath);
+
+ // Set the type of resource - file or archive
+ // archives are untarred at destination
+ // we don't need the jar file to be untarred for now
+ return createAmResource(destPath, LocalResourceType.FILE);
+ }
+
+ /**
+ * Submit the AM tar.gz resource referenced by the instance's cluster
+ * filesystem. Also, update the providerResources object with the new
+ * resource.
+ *
+ * @param providerResources
+ * the provider resource map to be updated
+ * @throws IOException
+ * trouble copying to HDFS
+ */
+ public void submitTarGzipAndUpdate(
+ Map<String, LocalResource> providerResources) throws IOException,
+ BadClusterStateException {
+ Path dependencyLibTarGzip = getDependencyTarGzip();
+ LocalResource lc = createAmResource(dependencyLibTarGzip,
+ LocalResourceType.ARCHIVE);
+ providerResources.put(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK, lc);
+ }
+
+ public void copyLocalFileToHdfs(File localPath,
+ Path destPath, FsPermission fp)
+ throws IOException {
+ if (localPath == null || destPath == null) {
+ throw new IOException("Either localPath or destPath is null");
+ }
+ fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
+ "000");
+ fileSystem.mkdirs(destPath.getParent(), fp);
+ log.info("Copying file {} to {}", localPath.toURI(),
+ fileSystem.getScheme() + ":/" + destPath.toUri());
+
+ fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
+ destPath);
+ // set file permissions of the destPath
+ fileSystem.setPermission(destPath, fp);
+ }
+
+ public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
+ throws IOException {
+ if (hdfsPath == null || destFile == null) {
+ throw new IOException("Either hdfsPath or destPath is null");
+ }
+ log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
+
+ Path destPath = new Path(destFile.getPath());
+ fileSystem.copyToLocalFile(hdfsPath, destPath);
+ }
+
+ /**
+ * list entries in a filesystem directory
+ *
+ * @param path directory
+ * @return a listing, one to a line
+ * @throws IOException
+ */
+ public String listFSDir(Path path) throws IOException {
+ FileStatus[] stats = fileSystem.listStatus(path);
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.getPath().toString())
+ .append("\t")
+ .append(stat.getLen())
+ .append("\n");
+ }
+ return builder.toString();
+ }
+
+ public String cat(Path path) throws IOException {
+ FileStatus status = fileSystem.getFileStatus(path);
+ byte[] b = new byte[(int) status.getLen()];
+ FSDataInputStream in = null;
+ try {
+ in = fileSystem.open(path);
+ int count = in.read(b);
+ return new String(b, 0, count, UTF_8);
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
new file mode 100644
index 0000000..6fadfd3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import java.io.Closeable;
+
+/**
+ * A duration in milliseconds. This class can be used
+ * to count time, and to be polled to see if a time limit has
+ * passed.
+ */
+public class Duration implements Closeable {
+ public long start, finish;
+ public final long limit;
+
+ /**
+ * Create a duration instance with a limit of 0
+ */
+ public Duration() {
+ this(0);
+ }
+
+ /**
+ * Create a duration with a limit specified in millis
+ * @param limit duration in milliseconds
+ */
+ public Duration(long limit) {
+ this.limit = limit;
+ }
+
+ /**
+ * Start
+ * @return self
+ */
+ public Duration start() {
+ start = now();
+ return this;
+ }
+
+ /**
+ * The close operation relays to {@link #finish()}.
+ * Implementing it allows Duration instances to be automatically
+ * finish()'d in Java7 try blocks for when used in measuring durations.
+ */
+ @Override
+ public final void close() {
+ finish();
+ }
+
+ public void finish() {
+ finish = now();
+ }
+
+ protected long now() {
+ return System.nanoTime()/1000000;
+ }
+
+ public long getInterval() {
+ return finish - start;
+ }
+
+ /**
+ * return true if the limit has been exceeded
+ * @return true if a limit was set and the current time
+ * exceeds it.
+ */
+ public boolean getLimitExceeded() {
+ return limit >= 0 && ((now() - start) > limit);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Duration");
+ if (finish >= start) {
+ builder.append(" finished at ").append(getInterval()).append(" millis;");
+ } else {
+ if (start > 0) {
+ builder.append(" started but not yet finished;");
+ } else {
+ builder.append(" unstarted;");
+ }
+ }
+ if (limit > 0) {
+ builder.append(" limit: ").append(limit).append(" millis");
+ if (getLimitExceeded()) {
+ builder.append(" - exceeded");
+ }
+ }
+ return builder.toString();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/58] [abbrv] hadoop git commit: Rebase onto trunk - fix some
issues. Contributed by Rohith Sharma K S
Posted by ji...@apache.org.
Rebase onto trunk - fix some issues. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eed9e3e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eed9e3e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eed9e3e8
Branch: refs/heads/yarn-native-services
Commit: eed9e3e8f4724ca42e0140754b8fe598b6472747
Parents: e660e52
Author: Jian He <ji...@apache.org>
Authored: Mon Sep 4 22:07:24 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:14 2017 -0800
----------------------------------------------------------------------
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop-yarn-services-core/pom.xml | 2 +-
.../ServiceTimelinePublisher.java | 32 ++++++++++++--------
.../TestServiceTimelinePublisher.java | 2 --
.../hadoop-yarn-services/pom.xml | 2 +-
.../src/main/webapp/app/routes/yarn-services.js | 2 +-
6 files changed, 23 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 1077ccd..74d9681 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications</artifactId>
- <version>3.0.0-beta1-SNAPSHOT</version>
+ <version>3.1.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-yarn-services-api</artifactId>
<name>Apache Hadoop YARN Services API</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index d9b7adb..fb07edc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services</artifactId>
- <version>3.0.0-beta1-SNAPSHOT</version>
+ <version>3.1.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-yarn-services-core</artifactId>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index 5e65ad9..9e7d8e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -66,6 +66,7 @@ public class ServiceTimelinePublisher extends CompositeService {
protected void serviceInit(org.apache.hadoop.conf.Configuration configuration)
throws Exception {
addService(timelineClient);
+ super.serviceInit(configuration);
}
@@ -95,8 +96,8 @@ public class ServiceTimelinePublisher extends CompositeService {
// create info keys
Map<String, Object> entityInfos = new HashMap<String, Object>();
entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName());
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- service.getState().toString());
+// entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+// service.getState().toString());
entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
currentTimeMillis);
entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
@@ -232,18 +233,23 @@ public class ServiceTimelinePublisher extends CompositeService {
// create info keys
Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_ID,
- component.getArtifact().getId());
- entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_TYPE,
- component.getArtifact().getType().toString());
- if (component.getResource().getProfile() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_PROFILE,
- component.getResource().getProfile());
+ if (component.getArtifact() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_ID,
+ component.getArtifact().getId());
+ entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_TYPE,
+ component.getArtifact().getType().toString());
+ }
+
+ if (component.getResource() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_CPU,
+ component.getResource().getCpus());
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_MEMORY,
+ component.getResource().getMemory());
+ if (component.getResource().getProfile() != null) {
+ entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_PROFILE,
+ component.getResource().getProfile());
+ }
}
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_CPU,
- component.getResource().getCpus());
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_MEMORY,
- component.getResource().getMemory());
if (component.getLaunchCommand() != null) {
entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_COMMAND,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
index b742553..e7c7600 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -82,9 +82,7 @@ public class TestServiceTimelinePublisher {
timelineClient =
new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
- timelineClient.init(config);
serviceTimelinePublisher.init(config);
- timelineClient.start();
serviceTimelinePublisher.start();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
index 1233804..716fdb7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-beta1-SNAPSHOT</version>
+ <version>3.1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-services</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
index 0f74f2f..a6535ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-services.js
@@ -23,7 +23,7 @@ export default AbstractRoute.extend({
model() {
return Ember.RSVP.hash({
apps: this.store.query('yarn-app', {
- applicationTypes: "yarn-native-service"
+ applicationTypes: "yarn-service"
}),
});
},
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
new file mode 100644
index 0000000..c64b1b5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class Error {
+
+ private Integer code = null;
+ private String message = null;
+ private String fields = null;
+
+ /**
+ **/
+ public Error code(Integer code) {
+ this.code = code;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("code")
+ public Integer getCode() {
+ return code;
+ }
+
+ public void setCode(Integer code) {
+ this.code = code;
+ }
+
+ /**
+ **/
+ public Error message(String message) {
+ this.message = message;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("message")
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ /**
+ **/
+ public Error fields(String fields) {
+ this.fields = fields;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("fields")
+ public String getFields() {
+ return fields;
+ }
+
+ public void setFields(String fields) {
+ this.fields = fields;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Error error = (Error) o;
+ return Objects.equals(this.code, error.code)
+ && Objects.equals(this.message, error.message)
+ && Objects.equals(this.fields, error.fields);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(code, message, fields);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Error {\n");
+
+ sb.append(" code: ").append(toIndentedString(code)).append("\n");
+ sb.append(" message: ").append(toIndentedString(message)).append("\n");
+ sb.append(" fields: ").append(toIndentedString(fields)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
new file mode 100644
index 0000000..6f6fe6f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Placement policy of an instance of an service. This feature is in the
+ * works in YARN-4902.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Placement policy of an instance of an service. This feature is in the works in YARN-4902.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class PlacementPolicy implements Serializable {
+ private static final long serialVersionUID = 4341110649551172231L;
+
+ private String label = null;
+
+ /**
+ * Assigns a service to a named partition of the cluster where the service
+ * desires to run (optional). If not specified all services are submitted to
+ * a default label of the service owner. One or more labels can be setup for
+ * each service owner account with required constraints like no-preemption,
+ * sla-99999, preemption-ok, etc.
+ **/
+ public PlacementPolicy label(String label) {
+ this.label = label;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.")
+ @JsonProperty("label")
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PlacementPolicy placementPolicy = (PlacementPolicy) o;
+ return Objects.equals(this.label, placementPolicy.label);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(label);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class PlacementPolicy {\n");
+
+ sb.append(" label: ").append(toIndentedString(label)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
new file mode 100644
index 0000000..1a25a4c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A custom command or a pluggable helper container to determine the readiness
+ * of a container of a component. Readiness for every service is different.
+ * Hence the need for a simple interface, with scope to support advanced
+ * usecases.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class ReadinessCheck implements Serializable {
+ private static final long serialVersionUID = -3836839816887186801L;
+
+ public enum TypeEnum {
+ HTTP("HTTP"),
+ PORT("PORT");
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = null;
+ private Map<String, String> props = new HashMap<String, String>();
+ private Artifact artifact = null;
+
+ /**
+ * E.g. HTTP (YARN will perform a simple REST call at a regular interval and
+ * expect a 204 No content).
+ **/
+ public ReadinessCheck type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ public ReadinessCheck props(Map<String, String> props) {
+ this.props = props;
+ return this;
+ }
+
+ public ReadinessCheck putPropsItem(String key, String propsItem) {
+ this.props.put(key, propsItem);
+ return this;
+ }
+
+ /**
+ * A blob of key value pairs that will be used to configure the check.
+ * @return props
+ **/
+ @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be used to configure the check.")
+ public Map<String, String> getProps() {
+ return props;
+ }
+
+ public void setProps(Map<String, String> props) {
+ this.props = props;
+ }
+
+ /**
+ * Artifact of the pluggable readiness check helper container (optional). If
+ * specified, this helper container typically hosts the http uri and
+ * encapsulates the complex scripts required to perform actual container
+ * readiness check. At the end it is expected to respond a 204 No content just
+ * like the simplified use case. This pluggable framework benefits service
+ * owners who can run services without any packaging modifications. Note,
+ * artifacts of type docker only is supported for now.
+ **/
+ public ReadinessCheck artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReadinessCheck readinessCheck = (ReadinessCheck) o;
+ return Objects.equals(this.type, readinessCheck.type) &&
+ Objects.equals(this.props, readinessCheck.props) &&
+ Objects.equals(this.artifact, readinessCheck.artifact);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, props, artifact);
+ }
+
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ReadinessCheck {\n");
+
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
new file mode 100644
index 0000000..cec9de9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Resource determines the amount of resources (vcores, memory, network, etc.)
+ * usable by a container. This field determines the resource to be applied for
+ * all the containers of a component or service. The resource specified at
+ * the service (or global) level can be overriden at the component level. Only one
+ * of profile OR cpu & memory are expected. It raises a validation
+ * exception otherwise.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class Resource extends BaseResource implements Cloneable {
+ private static final long serialVersionUID = -6431667797380250037L;
+
+ private String profile = null;
+ private Integer cpus = 1;
+ private String memory = null;
+
+ /**
+ * Each resource profile has a unique id which is associated with a
+ * cluster-level predefined memory, cpus, etc.
+ **/
+ public Resource profile(String profile) {
+ this.profile = profile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.")
+ @JsonProperty("profile")
+ public String getProfile() {
+ return profile;
+ }
+
+ public void setProfile(String profile) {
+ this.profile = profile;
+ }
+
+ /**
+ * Amount of vcores allocated to each container (optional but overrides cpus
+ * in profile if specified).
+ **/
+ public Resource cpus(Integer cpus) {
+ this.cpus = cpus;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).")
+ @JsonProperty("cpus")
+ public Integer getCpus() {
+ return cpus;
+ }
+
+ public void setCpus(Integer cpus) {
+ this.cpus = cpus;
+ }
+
+ /**
+ * Amount of memory allocated to each container (optional but overrides memory
+ * in profile if specified). Currently accepts only an integer value and
+ * default unit is in MB.
+ **/
+ public Resource memory(String memory) {
+ this.memory = memory;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.")
+ @JsonProperty("memory")
+ public String getMemory() {
+ return memory;
+ }
+
+ public void setMemory(String memory) {
+ this.memory = memory;
+ }
+
+ public long getMemoryMB() {
+ if (this.memory == null) {
+ return 0;
+ }
+ return Long.parseLong(memory);
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Resource resource = (Resource) o;
+ return Objects.equals(this.profile, resource.profile)
+ && Objects.equals(this.cpus, resource.cpus)
+ && Objects.equals(this.memory, resource.memory);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(profile, cpus, memory);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Resource {\n");
+
+ sb.append(" profile: ").append(toIndentedString(profile)).append("\n");
+ sb.append(" cpus: ").append(toIndentedString(cpus)).append("\n");
+ sb.append(" memory: ").append(toIndentedString(memory)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
new file mode 100644
index 0000000..f3bbaa0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * An Service resource has the following attributes.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "An Service resource has the following attributes.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
+ "lifetime", "containers" })
+public class Service extends BaseResource {
+ private static final long serialVersionUID = -4491694636566094885L;
+
+ private String name = null;
+ private String id = null;
+ private Artifact artifact = null;
+ private Resource resource = null;
+ private String launchCommand = null;
+ private Date launchTime = null;
+ private Long numberOfContainers = null;
+ private Long numberOfRunningContainers = null;
+ private Long lifetime = null;
+ private PlacementPolicy placementPolicy = null;
+ private List<Component> components = new ArrayList<>();
+ private Configuration configuration = new Configuration();
+ private List<Container> containers = new ArrayList<>();
+ private ServiceState state = null;
+ private Map<String, String> quicklinks = new HashMap<>();
+ private String queue = null;
+
+ /**
+ * A unique service name.
+ **/
+ public Service name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "A unique service name.")
+ @JsonProperty("name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * A unique service id.
+ **/
+ public Service id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A unique service id.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * Artifact of single-component services. Mandatory if components
+ * attribute is not specified.
+ **/
+ public Service artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of single-component services. Mandatory if components attribute is not specified.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * Resource of single-component services or the global default for
+ * multi-component services. Mandatory if it is a single-component
+ * service and if cpus and memory are not specified at the Service
+ * level.
+ **/
+ public Service resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource of single-component services or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * The custom launch command of an service component (optional). If not
+ * specified for services with docker images say, it will default to the
+ * default start command of the image. If there is a single component in this
+ * service, you can specify this without the need to have a 'components'
+ * section.
+ **/
+ public Service launchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The custom launch command of an service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.")
+ @JsonProperty("launch_command")
+ public String getLaunchCommand() {
+ return launchCommand;
+ }
+
+ @XmlElement(name = "launch_command")
+ public void setLaunchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ }
+
+ /**
+ * The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
+ **/
+ public Service launchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.")
+ @JsonProperty("launch_time")
+ public Date getLaunchTime() {
+ return launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ @XmlElement(name = "launch_time")
+ public void setLaunchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ /**
+ * Number of containers for each component in the service. Each
+ * component can further override this service-level global default.
+ **/
+ public Service numberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Number of containers for each component in the service. Each component can further override this service-level global default.")
+ @JsonProperty("number_of_containers")
+ public Long getNumberOfContainers() {
+ return numberOfContainers;
+ }
+
+ @XmlElement(name = "number_of_containers")
+ public void setNumberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ }
+
+ /**
+ * In get response this provides the total number of running containers for
+ * this service (across all components) at the time of request. Note, a
+ * subsequent request can return a different number as and when more
+ * containers get allocated until it reaches the total number of containers or
+ * if a flex request has been made between the two requests.
+ **/
+ public Service numberOfRunningContainers(Long numberOfRunningContainers) {
+ this.numberOfRunningContainers = numberOfRunningContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
+ @JsonProperty("number_of_running_containers")
+ public Long getNumberOfRunningContainers() {
+ return numberOfRunningContainers;
+ }
+
+ @XmlElement(name = "number_of_running_containers")
+ public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
+ this.numberOfRunningContainers = numberOfRunningContainers;
+ }
+
+ /**
+ * Life time (in seconds) of the service from the time it reaches the
+ * RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For
+ * unlimited lifetime do not set a lifetime value.
+ **/
+ public Service lifetime(Long lifetime) {
+ this.lifetime = lifetime;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Life time (in seconds) of the service from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.")
+ @JsonProperty("lifetime")
+ public Long getLifetime() {
+ return lifetime;
+ }
+
+ public void setLifetime(Long lifetime) {
+ this.lifetime = lifetime;
+ }
+
+ /**
+ * Advanced scheduling and placement policies (optional). If not specified, it
+ * defaults to the default placement policy of the service owner. The design of
+ * placement policies are in the works. It is not very clear at this point,
+ * how policies in conjunction with labels be exposed to service owners.
+ * This is a placeholder for now. The advanced structure of this attribute
+ * will be determined by YARN-4902.
+ **/
+ public Service placementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
+ @JsonProperty("placement_policy")
+ public PlacementPolicy getPlacementPolicy() {
+ return placementPolicy;
+ }
+
+ @XmlElement(name = "placement_policy")
+ public void setPlacementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ }
+
+ /**
+ * Components of an service.
+ **/
+ public Service components(List<Component> components) {
+ this.components = components;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Components of an service.")
+ @JsonProperty("components")
+ public List<Component> getComponents() {
+ return components;
+ }
+
+ public void setComponents(List<Component> components) {
+ this.components = components;
+ }
+
+ public void addComponent(Component component) {
+ components.add(component);
+ }
+
+ public Component getComponent(String name) {
+ for (Component component : components) {
+ if (component.getName().equals(name)) {
+ return component;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Config properties of an service. Configurations provided at the
+ * service/global level are available to all the components. Specific
+ * properties can be overridden at the component level.
+ **/
+ public Service configuration(Configuration configuration) {
+ this.configuration = configuration;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config properties of an service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.")
+ @JsonProperty("configuration")
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * Containers of a started service. Specifying a value for this attribute
+ * for the POST payload raises a validation error. This blob is available only
+ * in the GET response of a started service.
+ **/
+ public Service containers(List<Container> containers) {
+ this.containers = containers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
+ @JsonProperty("containers")
+ public List<Container> getContainers() {
+ return containers;
+ }
+
+ public void setContainers(List<Container> containers) {
+ this.containers = containers;
+ }
+
+ public void addContainer(Container container) {
+ this.containers.add(container);
+ }
+
+ /**
+ * State of the service. Specifying a value for this attribute for the
+ * POST payload raises a validation error. This attribute is available only in
+ * the GET response of a started service.
+ **/
+ public Service state(ServiceState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.")
+ @JsonProperty("state")
+ public ServiceState getState() {
+ return state;
+ }
+
+ public void setState(ServiceState state) {
+ this.state = state;
+ }
+
+ /**
+ * A blob of key-value pairs of quicklinks to be exported for an service.
+ **/
+ public Service quicklinks(Map<String, String> quicklinks) {
+ this.quicklinks = quicklinks;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an service.")
+ @JsonProperty("quicklinks")
+ public Map<String, String> getQuicklinks() {
+ return quicklinks;
+ }
+
+ public void setQuicklinks(Map<String, String> quicklinks) {
+ this.quicklinks = quicklinks;
+ }
+
+ /**
+ * The YARN queue that this service should be submitted to.
+ **/
+ public Service queue(String queue) {
+ this.queue = queue;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The YARN queue that this service should be submitted to.")
+ @JsonProperty("queue")
+ public String getQueue() {
+ return queue;
+ }
+
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Service service = (Service) o;
+ return Objects.equals(this.name, service.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Service {\n");
+
+ sb.append(" name: ").append(toIndentedString(name)).append("\n");
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
+ .append("\n");
+ sb.append(" launchTime: ").append(toIndentedString(launchTime))
+ .append("\n");
+ sb.append(" numberOfContainers: ")
+ .append(toIndentedString(numberOfContainers)).append("\n");
+ sb.append(" numberOfRunningContainers: ")
+ .append(toIndentedString(numberOfRunningContainers)).append("\n");
+ sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n");
+ sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
+ .append("\n");
+ sb.append(" components: ").append(toIndentedString(components))
+ .append("\n");
+ sb.append(" configuration: ").append(toIndentedString(configuration))
+ .append("\n");
+ sb.append(" containers: ").append(toIndentedString(containers))
+ .append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
+ .append("\n");
+ sb.append(" queue: ").append(toIndentedString(queue)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
new file mode 100644
index 0000000..a4509bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current state of an service.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "The current state of an service.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public enum ServiceState {
+ ACCEPTED, STARTED, READY, STOPPED, FAILED;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceStatus.java
new file mode 100644
index 0000000..2cee23c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceStatus.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current status of a submitted service, returned as a response to the
+ * GET API.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "The current status of a submitted service, returned as a response to the GET API.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ServiceStatus extends BaseResource {
+ private static final long serialVersionUID = -3469885905347851034L;
+
+ private String diagnostics = null;
+ private ServiceState state = null;
+ private Integer code = null;
+
+ /**
+ * Diagnostic information (if any) for the reason of the current state of the
+ * service. It typically has a non-null value, if the service is in a
+ * non-running state.
+ **/
+ public ServiceStatus diagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.")
+ @JsonProperty("diagnostics")
+ public String getDiagnostics() {
+ return diagnostics;
+ }
+
+ public void setDiagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ }
+
+ /**
+ * Service state.
+ **/
+ public ServiceStatus state(ServiceState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Service state.")
+ @JsonProperty("state")
+ public ServiceState getState() {
+ return state;
+ }
+
+ public void setState(ServiceState state) {
+ this.state = state;
+ }
+
+ /**
+ * An error code specific to a scenario which service owners should be able to use
+ * to understand the failure in addition to the diagnostic information.
+ **/
+ public ServiceStatus code(Integer code) {
+ this.code = code;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.")
+ @JsonProperty("code")
+ public Integer getCode() {
+ return code;
+ }
+
+ public void setCode(Integer code) {
+ this.code = code;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ServiceStatus serviceStatus = (ServiceStatus) o;
+ return Objects.equals(this.diagnostics, serviceStatus.diagnostics)
+ && Objects.equals(this.state, serviceStatus.state)
+ && Objects.equals(this.code, serviceStatus.code);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(diagnostics, state, code);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ServiceStatus {\n");
+
+ sb.append(" diagnostics: ").append(toIndentedString(diagnostics))
+ .append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" code: ").append(toIndentedString(code)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
new file mode 100644
index 0000000..e17c0c4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.client.ServerProxy;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+
+import java.net.InetSocketAddress;
+
+import static org.apache.hadoop.io.retry.RetryPolicies.TRY_ONCE_THEN_FAIL;
+
+public class ClientAMProxy extends ServerProxy{
+
+ public static <T> T createProxy(final Configuration conf,
+ final Class<T> protocol, final UserGroupInformation ugi,
+ final YarnRPC rpc, final InetSocketAddress serverAddress) {
+ Configuration confClone = new Configuration(conf);
+ confClone.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
+ confClone.setInt(CommonConfigurationKeysPublic.
+ IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
+ RetryPolicy retryPolicy;
+
+ if (conf.getLong(YarnServiceConf.CLIENT_AM_RETRY_MAX_WAIT_MS, 0) == 0) {
+ // by default no retry
+ retryPolicy = TRY_ONCE_THEN_FAIL;
+ } else {
+ retryPolicy =
+ createRetryPolicy(conf, YarnServiceConf.CLIENT_AM_RETRY_MAX_WAIT_MS,
+ 15 * 60 * 1000, YarnServiceConf.CLIENT_AM_RETRY_MAX_INTERVAL_MS,
+ 2 * 1000);
+ }
+ return createRetriableProxy(confClone, protocol, ugi, rpc, serverAddress,
+ retryPolicy);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
new file mode 100644
index 0000000..cb27021
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.*;
+
+public class ServiceCLI {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceClient.class);
+ protected ServiceClient client;
+
+ int exec(ClientArgs args) throws Throwable {
+ if (StringUtils.isEmpty(args.getAction())) {
+ System.out.println(args.usage());
+ return -1;
+ }
+ switch (args.getAction()) {
+ case ACTION_BUILD: // Upload app json onto hdfs
+ client.actionBuild(args.getActionBuildArgs());
+ break;
+ case ACTION_START: // start the app with the pre-uploaded app json on hdfs
+ client.actionStart(args.getClusterName());
+ break;
+ case ACTION_CREATE: // create == build + start
+ client.actionCreate(args.getActionCreateArgs());
+ break;
+ case ACTION_STATUS:
+ Service app = client.getStatus(args.getClusterName());
+ System.out.println(app);
+ break;
+ case ACTION_FLEX:
+ try {
+ client.actionFlexByCLI(args);
+ } catch (FileNotFoundException e) {
+ System.err.println(
+ args.getClusterName() + " doesn't exist: " + e.getMessage());
+ return -1;
+ }
+ break;
+ case ACTION_STOP:
+ client.actionStop(args.getClusterName(), false);
+ break;
+ case ACTION_DESTROY: // Destroy can happen only if app is already stopped
+ client.actionDestroy(args.getClusterName());
+ break;
+ case ACTION_DEPENDENCY: // upload dependency jars
+ client.actionDependency(args.getActionDependencyArgs());
+ break;
+ case ACTION_UPDATE:
+ client.updateLifetime(args.getClusterName(),
+ args.getActionUpdateArgs().lifetime);
+ break;
+ case ACTION_HELP:
+ LOG.info(args.usage());
+ break;
+ default:
+ LOG.info("NOT IMPLEMENTED: " + args.getAction());
+ LOG.info(args.usage());
+ return -1;
+ }
+ return 0;
+ }
+
+ public ServiceCLI() {
+ createServiceClient();
+ }
+
+ protected void createServiceClient() {
+ client = new ServiceClient();
+ client.init(new YarnConfiguration());
+ client.start();
+ }
+
+ public static void main(String[] args) throws Throwable {
+ ClientArgs clientArgs = new ClientArgs(args);
+ try {
+ clientArgs.parse();
+ } catch (BadCommandArgumentsException e) {
+ System.err.println(e.getMessage());
+ System.exit(-1);
+ }
+ ServiceCLI cli = new ServiceCLI();
+ int res = cli.exec(clientArgs);
+ System.exit(res);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
deleted file mode 100644
index 405f690..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-
-/**
- * Serialized form of an application report which can be persisted
- * and then parsed. It can not be converted back into a
- * real YARN application report
- *
- * Useful for testing
- */
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-
-public class SerializedApplicationReport {
-
- public String applicationId;
- public String applicationAttemptId;
- public String name;
- public String applicationType;
- public String user;
- public String queue;
- public String host;
- public Integer rpcPort;
- public String state;
- public String diagnostics;
- public String url;
- /**
- * This value is non-null only when a report is generated from a submission context.
- * The YARN {@link ApplicationReport} structure does not propagate this value
- * from the RM.
- */
- public Long submitTime;
- public Long startTime;
- public Long finishTime;
- public String finalStatus;
- public String origTrackingUrl;
- public Float progress;
-
- public SerializedApplicationReport() {
- }
-
- public SerializedApplicationReport(ApplicationReport report) {
- this.applicationId = report.getApplicationId().toString();
- ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
- this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A";
- this.name = report.getName();
- this.applicationType = report.getApplicationType();
- this.user = report.getUser();
- this.queue = report.getQueue();
- this.host = report.getHost();
- this.rpcPort = report.getRpcPort();
- this.state = report.getYarnApplicationState().toString();
- this.diagnostics = report.getDiagnostics();
- this.startTime = report.getStartTime();
- this.finishTime = report.getFinishTime();
- FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
- this.finalStatus = appStatus == null ? "" : appStatus.toString();
- this.progress = report.getProgress();
- this.url = report.getTrackingUrl();
- this.origTrackingUrl= report.getOriginalTrackingUrl();
- }
-
- @Override
- public String toString() {
- try {
- return ApplicationReportSerDeser.toString(this);
- } catch (IOException e) {
- return super.toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
deleted file mode 100644
index 21cb049..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ /dev/null
@@ -1,443 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.MonitorUtils;
-import org.apache.hadoop.yarn.service.conf.RestApiConstants;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class ServiceApiUtil {
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceApiUtil.class);
- public static JsonSerDeser<Application> jsonSerDeser =
- new JsonSerDeser<>(Application.class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
- private static final PatternValidator compNamePattern
- = new PatternValidator("[a-z][a-z0-9-]*");
-
- @VisibleForTesting
- public static void setJsonSerDeser(JsonSerDeser jsd) {
- jsonSerDeser = jsd;
- }
-
- @VisibleForTesting
- public static void validateAndResolveApplication(Application application,
- SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf) throws
- IOException {
- boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
- RegistryConstants.DEFAULT_DNS_ENABLED);
- if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
- .MAX_FQDN_LABEL_LENGTH) {
- throw new IllegalArgumentException(RestApiErrorMessages
- .ERROR_USER_NAME_INVALID);
- }
- if (StringUtils.isEmpty(application.getName())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
- }
- if (!SliderUtils.isClusternameValid(application.getName()) || (dnsEnabled
- && application.getName().length() > RegistryConstants
- .MAX_FQDN_LABEL_LENGTH)) {
- throw new IllegalArgumentException(String.format(
- RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT,
- application.getName()));
- }
-
- // If the application has no components do top-level checks
- if (!hasComponent(application)) {
- // If artifact is of type APPLICATION, read other application components
- if (application.getArtifact() != null && application.getArtifact()
- .getType() == Artifact.TypeEnum.APPLICATION) {
- if (StringUtils.isEmpty(application.getArtifact().getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- Application otherApplication = loadApplication(fs,
- application.getArtifact().getId());
- application.setComponents(otherApplication.getComponents());
- application.setArtifact(null);
- SliderUtils.mergeMapsIgnoreDuplicateKeys(application.getQuicklinks(),
- otherApplication.getQuicklinks());
- } else {
- // Since it is a simple app with no components, create a default
- // component
- Component comp = createDefaultComponent(application);
- validateComponent(comp, fs.getFileSystem());
- application.getComponents().add(comp);
- if (application.getLifetime() == null) {
- application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
- }
- return;
- }
- }
-
- // Validate there are no component name collisions (collisions are not
- // currently supported) and add any components from external applications
- // TODO allow name collisions? see AppState#roles
- // TODO or add prefix to external component names?
- Configuration globalConf = application.getConfiguration();
- Set<String> componentNames = new HashSet<>();
- List<Component> componentsToRemove = new ArrayList<>();
- List<Component> componentsToAdd = new ArrayList<>();
- for (Component comp : application.getComponents()) {
- int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
- maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length();
- if (dnsEnabled && comp.getName().length() > maxCompLength) {
- throw new IllegalArgumentException(String.format(RestApiErrorMessages
- .ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName()));
- }
- if (componentNames.contains(comp.getName())) {
- throw new IllegalArgumentException("Component name collision: " +
- comp.getName());
- }
- // If artifact is of type APPLICATION (which cannot be filled from
- // global), read external application and add its components to this
- // application
- if (comp.getArtifact() != null && comp.getArtifact().getType() ==
- Artifact.TypeEnum.APPLICATION) {
- if (StringUtils.isEmpty(comp.getArtifact().getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- LOG.info("Marking {} for removal", comp.getName());
- componentsToRemove.add(comp);
- List<Component> externalComponents = getApplicationComponents(fs,
- comp.getArtifact().getId());
- for (Component c : externalComponents) {
- Component override = application.getComponent(c.getName());
- if (override != null && override.getArtifact() == null) {
- // allow properties from external components to be overridden /
- // augmented by properties in this component, except for artifact
- // which must be read from external component
- override.mergeFrom(c);
- LOG.info("Merging external component {} from external {}", c
- .getName(), comp.getName());
- } else {
- if (componentNames.contains(c.getName())) {
- throw new IllegalArgumentException("Component name collision: " +
- c.getName());
- }
- componentNames.add(c.getName());
- componentsToAdd.add(c);
- LOG.info("Adding component {} from external {}", c.getName(),
- comp.getName());
- }
- }
- } else {
- // otherwise handle as a normal component
- componentNames.add(comp.getName());
- // configuration
- comp.getConfiguration().mergeFrom(globalConf);
- }
- }
- application.getComponents().removeAll(componentsToRemove);
- application.getComponents().addAll(componentsToAdd);
-
- // Validate components and let global values take effect if component level
- // values are not provided
- Artifact globalArtifact = application.getArtifact();
- Resource globalResource = application.getResource();
- Long globalNumberOfContainers = application.getNumberOfContainers();
- String globalLaunchCommand = application.getLaunchCommand();
- for (Component comp : application.getComponents()) {
- // fill in global artifact unless it is type APPLICATION
- if (comp.getArtifact() == null && application.getArtifact() != null
- && application.getArtifact().getType() != Artifact.TypeEnum
- .APPLICATION) {
- comp.setArtifact(globalArtifact);
- }
- // fill in global resource
- if (comp.getResource() == null) {
- comp.setResource(globalResource);
- }
- // fill in global container count
- if (comp.getNumberOfContainers() == null) {
- comp.setNumberOfContainers(globalNumberOfContainers);
- }
- // fill in global launch command
- if (comp.getLaunchCommand() == null) {
- comp.setLaunchCommand(globalLaunchCommand);
- }
- // validate dependency existence
- if (comp.getDependencies() != null) {
- for (String dependency : comp.getDependencies()) {
- if (!componentNames.contains(dependency)) {
- throw new IllegalArgumentException(String.format(
- RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency,
- comp.getName()));
- }
- }
- }
- validateComponent(comp, fs.getFileSystem());
- }
-
- // validate dependency tree
- sortByDependencies(application.getComponents());
-
- // Application lifetime if not specified, is set to unlimited lifetime
- if (application.getLifetime() == null) {
- application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
- }
- }
-
- public static void validateComponent(Component comp, FileSystem fs)
- throws IOException {
- validateCompName(comp.getName());
-
- AbstractClientProvider compClientProvider = ProviderFactory
- .getClientProvider(comp.getArtifact());
- compClientProvider.validateArtifact(comp.getArtifact(), fs);
-
- if (comp.getLaunchCommand() == null && (comp.getArtifact() == null || comp
- .getArtifact().getType() != Artifact.TypeEnum.DOCKER)) {
- throw new IllegalArgumentException(RestApiErrorMessages
- .ERROR_ABSENT_LAUNCH_COMMAND);
- }
-
- validateApplicationResource(comp.getResource(), comp);
-
- if (comp.getNumberOfContainers() == null
- || comp.getNumberOfContainers() < 0) {
- throw new IllegalArgumentException(String.format(
- RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID
- + ": " + comp.getNumberOfContainers(), comp.getName()));
- }
- compClientProvider.validateConfigFiles(comp.getConfiguration()
- .getFiles(), fs);
-
- MonitorUtils.getProbe(comp.getReadinessCheck());
- }
-
- // Check component name format and transform to lower case.
- public static void validateCompName(String compName) {
- if (StringUtils.isEmpty(compName)) {
- throw new IllegalArgumentException("Component name can not be empty");
- }
- // validate component name
- if (compName.contains("_")) {
- throw new IllegalArgumentException(
- "Invalid format for component name: " + compName
- + ", can not use '_' as DNS hostname does not allow underscore. Use '-' Instead. ");
- }
- compNamePattern.validate(compName);
- }
-
- @VisibleForTesting
- public static List<Component> getApplicationComponents(SliderFileSystem
- fs, String appName) throws IOException {
- return loadApplication(fs, appName).getComponents();
- }
-
- public static Application loadApplication(SliderFileSystem fs, String
- appName) throws IOException {
- Path appJson = getAppJsonPath(fs, appName);
- LOG.info("Loading application definition from " + appJson);
- return jsonSerDeser.load(fs.getFileSystem(), appJson);
- }
-
- public static Application loadApplicationFrom(SliderFileSystem fs,
- Path appDefPath) throws IOException {
- LOG.info("Loading application definition from " + appDefPath);
- return jsonSerDeser.load(fs.getFileSystem(), appDefPath);
- }
-
- public static Path getAppJsonPath(SliderFileSystem fs, String appName) {
- Path appDir = fs.buildClusterDirPath(appName);
- Path appJson = new Path(appDir, appName + ".json");
- return appJson;
- }
-
- private static void validateApplicationResource(Resource resource,
- Component comp) {
- // Only apps/components of type APPLICATION can skip resource requirement
- if (resource == null) {
- throw new IllegalArgumentException(
- comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String
- .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
- comp.getName()));
- }
- // One and only one of profile OR cpus & memory can be specified. Specifying
- // both raises validation error.
- if (StringUtils.isNotEmpty(resource.getProfile()) && (
- resource.getCpus() != null || StringUtils
- .isNotEmpty(resource.getMemory()))) {
- throw new IllegalArgumentException(comp == null ?
- RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED :
- String.format(
- RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
- comp.getName()));
- }
- // Currently resource profile is not supported yet, so we will raise
- // validation error if only resource profile is specified
- if (StringUtils.isNotEmpty(resource.getProfile())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET);
- }
-
- String memory = resource.getMemory();
- Integer cpus = resource.getCpus();
- if (StringUtils.isEmpty(memory)) {
- throw new IllegalArgumentException(
- comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID :
- String.format(
- RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
- comp.getName()));
- }
- if (cpus == null) {
- throw new IllegalArgumentException(
- comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID :
- String.format(
- RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID,
- comp.getName()));
- }
- if (cpus <= 0) {
- throw new IllegalArgumentException(comp == null ?
- RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String
- .format(
- RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
- comp.getName()));
- }
- }
-
- // check if comp mem size exceeds cluster limit
- public static void validateCompResourceSize(
- org.apache.hadoop.yarn.api.records.Resource maxResource,
- Application application) throws YarnException {
- for (Component component : application.getComponents()) {
- // only handle mem now.
- long mem = Long.parseLong(component.getResource().getMemory());
- if (mem > maxResource.getMemorySize()) {
- throw new YarnException(
- "Component " + component.getName() + " memory size (" + mem
- + ") is larger than configured max container memory size ("
- + maxResource.getMemorySize() + ")");
- }
- }
- }
-
- public static boolean hasComponent(Application application) {
- if (application.getComponents() == null || application.getComponents()
- .isEmpty()) {
- return false;
- }
- return true;
- }
-
- public static Component createDefaultComponent(Application app) {
- Component comp = new Component();
- comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME);
- comp.setArtifact(app.getArtifact());
- comp.setResource(app.getResource());
- comp.setNumberOfContainers(app.getNumberOfContainers());
- comp.setLaunchCommand(app.getLaunchCommand());
- comp.setConfiguration(app.getConfiguration());
- return comp;
- }
-
- public static Collection<Component> sortByDependencies(List<Component>
- components) {
- Map<String, Component> sortedComponents =
- sortByDependencies(components, null);
- return sortedComponents.values();
- }
-
- /**
- * Each internal call of sortByDependencies will identify all of the
- * components with the same dependency depth (the lowest depth that has not
- * been processed yet) and add them to the sortedComponents list, preserving
- * their original ordering in the components list.
- *
- * So the first time it is called, all components with no dependencies
- * (depth 0) will be identified. The next time it is called, all components
- * that have dependencies only on the the depth 0 components will be
- * identified (depth 1). This will be repeated until all components have
- * been added to the sortedComponents list. If no new components are
- * identified but the sortedComponents list is not complete, an error is
- * thrown.
- */
- private static Map<String, Component> sortByDependencies(List<Component>
- components, Map<String, Component> sortedComponents) {
- if (sortedComponents == null) {
- sortedComponents = new LinkedHashMap<>();
- }
-
- Map<String, Component> componentsToAdd = new LinkedHashMap<>();
- List<Component> componentsSkipped = new ArrayList<>();
- for (Component component : components) {
- String name = component.getName();
- if (sortedComponents.containsKey(name)) {
- continue;
- }
- boolean dependenciesAlreadySorted = true;
- if (!SliderUtils.isEmpty(component.getDependencies())) {
- for (String dependency : component.getDependencies()) {
- if (!sortedComponents.containsKey(dependency)) {
- dependenciesAlreadySorted = false;
- break;
- }
- }
- }
- if (dependenciesAlreadySorted) {
- componentsToAdd.put(name, component);
- } else {
- componentsSkipped.add(component);
- }
- }
-
- if (componentsToAdd.size() == 0) {
- throw new IllegalArgumentException(String.format(RestApiErrorMessages
- .ERROR_DEPENDENCY_CYCLE, componentsSkipped));
- }
- sortedComponents.putAll(componentsToAdd);
- if (sortedComponents.size() == components.size()) {
- return sortedComponents;
- }
- return sortByDependencies(components, sortedComponents);
- }
-
- public static String $(String s) {
- return "${" + s +"}";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
deleted file mode 100644
index 7440b11..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-
-
-public class ServiceRegistryUtils {
-
- /**
- * Base path for services
- */
- public static final String ZK_SERVICES = "services";
-
- /**
- * Base path for all Slider references
- */
- public static final String ZK_SLIDER = "slider";
- public static final String ZK_USERS = "users";
- public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
- public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
-
- /**
- * Get the registry path for an instance under the user's home node
- * @param instanceName application instance
- * @return a path to the registry location for this application instance.
- */
- public static String registryPathForInstance(String instanceName) {
- return RegistryUtils.servicePath(
- RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, instanceName
- );
- }
-
- /**
- * Build the path to a cluster; exists once the cluster has come up.
- * Even before that, a ZK watcher could wait for it.
- * @param username user
- * @param clustername name of the cluster
- * @return a strin
- */
- public static String mkClusterPath(String username, String clustername) {
- return mkSliderUserPath(username) + "/" + clustername;
- }
-
- /**
- * Build the path to a cluster; exists once the cluster has come up.
- * Even before that, a ZK watcher could wait for it.
- * @param username user
- * @return a string
- */
- public static String mkSliderUserPath(String username) {
- return SVC_SLIDER_USERS + "/" + username;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
deleted file mode 100644
index d6d664e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-/**
- * Extends Core Filesystem with operations to manipulate ClusterDescription
- * persistent state
- */
-public class SliderFileSystem extends CoreFileSystem {
-
- Path appDir = null;
-
- public SliderFileSystem(FileSystem fileSystem,
- Configuration configuration) {
- super(fileSystem, configuration);
- }
-
- public SliderFileSystem(Configuration configuration) throws IOException {
- super(configuration);
- }
-
- public void setAppDir(Path appDir) {
- this.appDir = appDir;
- }
-
- public Path getAppDir() {
- return this.appDir;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
deleted file mode 100644
index 415392a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ /dev/null
@@ -1,1699 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.zookeeper.server.util.KerberosUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.Serializable;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-/**
- * These are slider-specific Util methods
- */
-public final class SliderUtils {
-
- private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
-
- /**
- * Atomic bool to track whether or not process security has already been
- * turned on (prevents re-entrancy)
- */
- private static final AtomicBoolean processSecurityAlreadyInitialized =
- new AtomicBoolean(false);
- public static final String JAVA_SECURITY_KRB5_REALM =
- "java.security.krb5.realm";
- public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
-
- /**
- * Winutils
- */
- public static final String WINUTILS = "WINUTILS.EXE";
- /**
- * name of openssl program
- */
- public static final String OPENSSL = "openssl";
-
- /**
- * name of python program
- */
- public static final String PYTHON = "python";
-
- /**
- * type of docker standalone application
- */
- public static final String DOCKER = "docker";
- /**
- * type of docker on yarn application
- */
- public static final String DOCKER_YARN = "yarn_docker";
-
- public static final int NODE_LIST_LIMIT = 10;
-
- private SliderUtils() {
- }
-
- /**
- * Implementation of set-ness, groovy definition of true/false for a string
- * @param s string
- * @return true iff the string is neither null nor empty
- */
- public static boolean isUnset(String s) {
- return s == null || s.isEmpty();
- }
-
- public static boolean isSet(String s) {
- return !isUnset(s);
- }
-
- public static boolean isEmpty(Collection l) {
- return l == null || l.isEmpty();
- }
-
- /**
- * Probe for a collection existing and not being empty
- * @param l collection
- * @return true if the reference is valid and it contains entries
- */
-
- public static boolean isNotEmpty(Collection l) {
- return l != null && !l.isEmpty();
- }
-
- /**
- * Probe for a map existing and not being empty
- * @param m map
- * @return true if the reference is valid and it contains map entries
- */
- public static boolean isNotEmpty(Map m) {
- return m != null && !m.isEmpty();
- }
-
- /*
- * Validates whether num is an integer
- * @param num
- * @param msg the message to be shown in exception
- */
- @SuppressWarnings("ResultOfMethodCallIgnored")
- private static void validateNumber(String num, String msg) throws
- BadConfigException {
- try {
- Integer.parseInt(num);
- } catch (NumberFormatException nfe) {
- throw new BadConfigException(msg + num);
- }
- }
-
- /*
- * Translates the trailing JVM heapsize unit: g, G, m, M
- * This assumes designated unit of 'm'
- * @param heapsize
- * @return heapsize in MB
- */
- public static String translateTrailingHeapUnit(String heapsize) throws
- BadConfigException {
- String errMsg = "Bad heapsize: ";
- if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
- String num = heapsize.substring(0, heapsize.length() - 1);
- validateNumber(num, errMsg);
- return num;
- }
- if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
- String num = heapsize.substring(0, heapsize.length() - 1) + "000";
- validateNumber(num, errMsg);
- return num;
- }
- // check if specified heap size is a number
- validateNumber(heapsize, errMsg);
- return heapsize;
- }
-
- /**
- * recursive directory delete
- * @param dir dir to delete
- * @throws IOException on any problem
- */
- public static void deleteDirectoryTree(File dir) throws IOException {
- if (dir.exists()) {
- if (dir.isDirectory()) {
- log.info("Cleaning up {}", dir);
- //delete the children
- File[] files = dir.listFiles();
- if (files == null) {
- throw new IOException("listfiles() failed for " + dir);
- }
- for (File file : files) {
- log.info("deleting {}", file);
- if (!file.delete()) {
- log.warn("Unable to delete " + file);
- }
- }
- if (!dir.delete()) {
- log.warn("Unable to delete " + dir);
- }
- } else {
- throw new IOException("Not a directory " + dir);
- }
- } else {
- //not found, do nothing
- log.debug("No output dir yet");
- }
- }
-
- /**
- * Find a containing JAR
- * @param clazz class to find
- * @return the file
- * @throws IOException any IO problem, including the class not having a
- * classloader
- * @throws FileNotFoundException if the class did not resolve to a file
- */
- public static File findContainingJarOrFail(Class clazz) throws IOException {
- File localFile = SliderUtils.findContainingJar(clazz);
- if (null == localFile) {
- throw new FileNotFoundException("Could not find JAR containing " + clazz);
- }
- return localFile;
- }
-
-
- /**
- * Find a containing JAR
- * @param my_class class to find
- * @return the file or null if it is not found
- * @throws IOException any IO problem, including the class not having a
- * classloader
- */
- public static File findContainingJar(Class my_class) throws IOException {
- ClassLoader loader = my_class.getClassLoader();
- if (loader == null) {
- throw new IOException(
- "Class " + my_class + " does not have a classloader!");
- }
- String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
- Enumeration<URL> urlEnumeration = loader.getResources(class_file);
- for (; urlEnumeration.hasMoreElements(); ) {
- URL url = urlEnumeration.nextElement();
- if ("jar".equals(url.getProtocol())) {
- String toReturn = url.getPath();
- if (toReturn.startsWith("file:")) {
- toReturn = toReturn.substring("file:".length());
- }
- // URLDecoder is a misnamed class, since it actually decodes
- // x-www-form-urlencoded MIME type rather than actual
- // URL encoding (which the file path has). Therefore it would
- // decode +s to ' 's which is incorrect (spaces are actually
- // either unencoded or encoded as "%20"). Replace +s first, so
- // that they are kept sacred during the decoding process.
- toReturn = toReturn.replaceAll("\\+", "%2B");
- toReturn = URLDecoder.decode(toReturn, "UTF-8");
- String jarFilePath = toReturn.replaceAll("!.*$", "");
- return new File(jarFilePath);
- } else {
- log.info("could not locate JAR containing {} URL={}", my_class, url);
- }
- }
- return null;
- }
-
- public static void checkPort(String hostname, int port, int connectTimeout)
- throws IOException {
- InetSocketAddress addr = new InetSocketAddress(hostname, port);
- checkPort(hostname, addr, connectTimeout);
- }
-
- @SuppressWarnings("SocketOpenedButNotSafelyClosed")
- public static void checkPort(String name,
- InetSocketAddress address,
- int connectTimeout)
- throws IOException {
- try(Socket socket = new Socket()) {
- socket.connect(address, connectTimeout);
- } catch (Exception e) {
- throw new IOException("Failed to connect to " + name
- + " at " + address
- + " after " + connectTimeout + "milliseconds"
- + ": " + e,
- e);
- }
- }
-
- public static void checkURL(String name, String url, int timeout) throws
- IOException {
- InetSocketAddress address = NetUtils.createSocketAddr(url);
- checkPort(name, address, timeout);
- }
-
- /**
- * A required file
- * @param role role of the file (for errors)
- * @param filename the filename
- * @throws ExitUtil.ExitException if the file is missing
- * @return the file
- */
- public static File requiredFile(String filename, String role) throws
- IOException {
- if (filename.isEmpty()) {
- throw new ExitUtil.ExitException(-1, role + " file not defined");
- }
- File file = new File(filename);
- if (!file.exists()) {
- throw new ExitUtil.ExitException(-1,
- role + " file not found: " +
- file.getCanonicalPath());
- }
- return file;
- }
-
- private static final PatternValidator clusternamePattern
- = new PatternValidator("[a-z][a-z0-9_-]*");
-
- private static final PatternValidator compNamePattern
- = new PatternValidator("[a-z][a-z0-9-]*");
-
- public static void validateCompName(String compName) {
- compNamePattern.validate(compName);
- }
-
- /**
- * Normalize a cluster name then verify that it is valid
- * @param name proposed cluster name
- * @return true iff it is valid
- */
- public static boolean isClusternameValid(String name) {
- return name != null && clusternamePattern.matches(name);
- }
-
- public static boolean oldIsClusternameValid(String name) {
- if (name == null || name.isEmpty()) {
- return false;
- }
- int first = name.charAt(0);
- if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
- return false;
- }
-
- for (int i = 0; i < name.length(); i++) {
- int elt = (int) name.charAt(i);
- int t = Character.getType(elt);
- if (0 == (t & Character.LOWERCASE_LETTER)
- && 0 == (t & Character.DECIMAL_DIGIT_NUMBER)
- && elt != '-'
- && elt != '_') {
- return false;
- }
- if (!Character.isLetterOrDigit(elt) && elt != '-' && elt != '_') {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Copy a directory to a new FS -both paths must be qualified. If
- * a directory needs to be created, supplied permissions can override
- * the default values. Existing directories are not touched
- * @param conf conf file
- * @param srcDirPath src dir
- * @param destDirPath dest dir
- * @param permission permission for the dest directory; null means "default"
- * @return # of files copies
- */
- @SuppressWarnings("deprecation")
- public static int copyDirectory(Configuration conf,
- Path srcDirPath,
- Path destDirPath,
- FsPermission permission) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
- FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcDirPath)) {
- throw new FileNotFoundException("Source dir not found " + srcDirPath);
- }
- if (!srcFS.isDirectory(srcDirPath)) {
- throw new FileNotFoundException(
- "Source dir not a directory " + srcDirPath);
- }
- GlobFilter dotFilter = new GlobFilter("[!.]*");
- FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
- int srcFileCount = entries.length;
- if (srcFileCount == 0) {
- return 0;
- }
- if (permission == null) {
- permission = FsPermission.getDirDefault();
- }
- if (!destFS.exists(destDirPath)) {
- new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
- permission);
- }
- Path[] sourcePaths = new Path[srcFileCount];
- for (int i = 0; i < srcFileCount; i++) {
- FileStatus e = entries[i];
- Path srcFile = e.getPath();
- if (srcFS.isDirectory(srcFile)) {
- String msg = "Configuration dir " + srcDirPath
- + " contains a directory " + srcFile;
- log.warn(msg);
- throw new IOException(msg);
- }
- log.debug("copying src conf file {}", srcFile);
- sourcePaths[i] = srcFile;
- }
- log.debug("Copying {} files from {} to dest {}", srcFileCount,
- srcDirPath,
- destDirPath);
- FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
- return srcFileCount;
- }
-
- /**
- * Copy a file to a new FS -both paths must be qualified.
- * @param conf conf file
- * @param srcFile src file
- * @param destFile dest file
- */
- @SuppressWarnings("deprecation")
- public static void copy(Configuration conf,
- Path srcFile,
- Path destFile) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcFile)) {
- throw new FileNotFoundException("Source file not found " + srcFile);
- }
- if (!srcFS.isFile(srcFile)) {
- throw new FileNotFoundException(
- "Source file not a file " + srcFile);
- }
- FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
- FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
- }
-
- /**
- * Take a collection, return a list containing the string value of every
- * element in the collection.
- * @param c collection
- * @return a stringified list
- */
- public static List<String> collectionToStringList(Collection c) {
- List<String> l = new ArrayList<>(c.size());
- for (Object o : c) {
- l.add(o.toString());
- }
- return l;
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -including at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(Collection collection, String separator) {
- return join(collection, separator, true);
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(Collection collection,
- String separator,
- boolean trailing) {
- StringBuilder b = new StringBuilder();
- // fast return on empty collection
- if (collection.isEmpty()) {
- return trailing ? separator : "";
- }
- for (Object o : collection) {
- b.append(o);
- b.append(separator);
- }
- int length = separator.length();
- String s = b.toString();
- return (trailing || s.isEmpty()) ?
- s : (b.substring(0, b.length() - length));
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -including at the end
- * @param collection strings
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(String[] collection, String separator) {
- return join(collection, separator, true);
-
-
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection strings
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(String[] collection, String separator,
- boolean trailing) {
- return join(Arrays.asList(collection), separator, trailing);
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -except at the end
- * @param collection strings
- * @param separator separator string
- * @return the list
- */
- public static String joinWithInnerSeparator(String separator,
- Object... collection) {
- StringBuilder b = new StringBuilder();
- boolean first = true;
-
- for (Object o : collection) {
- if (first) {
- first = false;
- } else {
- b.append(separator);
- }
- b.append(o.toString());
- b.append(separator);
- }
- return b.toString();
- }
-
- /**
- * Resolve a mandatory environment variable
- * @param key env var
- * @return the resolved value
- * @throws BadClusterStateException
- */
- public static String mandatoryEnvVariable(String key) throws
- BadClusterStateException {
- String v = System.getenv(key);
- if (v == null) {
- throw new BadClusterStateException("Missing Environment variable " + key);
- }
- return v;
- }
-
- public static String appReportToString(ApplicationReport r,
- String separator) {
- StringBuilder builder = new StringBuilder(512);
- builder.append("application ")
- .append(
- r.getName())
- .append("/")
- .append(r.getApplicationType())
- .append(separator);
- Set<String> tags = r.getApplicationTags();
- if (!tags.isEmpty()) {
- for (String tag : tags) {
- builder.append(tag).append(separator);
- }
- }
- DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
- dateFormat.setTimeZone(TimeZone.getDefault());
- builder.append("state: ").append(r.getYarnApplicationState());
- String trackingUrl = r.getTrackingUrl();
- if (isSet(trackingUrl)) {
- builder.append(separator).append("URL: ").append(trackingUrl);
- }
- builder.append(separator)
- .append("Started: ")
- .append(dateFormat.format(new Date(r.getStartTime())));
- long finishTime = r.getFinishTime();
- if (finishTime > 0) {
- builder.append(separator)
- .append("Finished: ")
- .append(dateFormat.format(new Date(finishTime)));
- }
- String rpcHost = r.getHost();
- if (!isSet(rpcHost)) {
- builder.append(separator)
- .append("RPC :")
- .append(rpcHost)
- .append(':')
- .append(r.getRpcPort());
- }
- String diagnostics = r.getDiagnostics();
- if (!isSet(diagnostics)) {
- builder.append(separator).append("Diagnostics :").append(diagnostics);
- }
- return builder.toString();
- }
-
- /**
- * Filter a string value given a single filter
- *
- * @param value
- * the string value to check
- * @param filter
- * a single string filter
- * @return return true if value should be trapped, false if it should be let
- * through
- */
- public static boolean filter(String value, String filter) {
- return !(StringUtils.isEmpty(filter) || filter.equals(value));
- }
-
- /**
- * Filter a string value given a set of filters
- *
- * @param value
- * the string value to check
- * @param filters
- * a set of string filters
- * @return return true if value should be trapped, false if it should be let
- * through
- */
- public static boolean filter(String value, Set<String> filters) {
- return !(filters.isEmpty() || filters.contains(value));
- }
-
- /**
- * Sorts the given list of application reports, most recently started
- * or finished instance first.
- *
- * @param instances list of instances
- */
- public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
- Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
- }
-
- /**
- * Sorts the given list of application reports
- * Finished instances are ordered by finished time and running/accepted instances are
- * ordered by start time
- * Finally Instance are order by finished instances coming after running instances
- *
- * @param instances list of instances
- */
- public static void sortApplicationReport(List<ApplicationReport> instances) {
- if (instances.size() <= 1) {
- return;
- }
- List<ApplicationReport> nonLiveInstance =
- new ArrayList<>(instances.size());
- List<ApplicationReport> liveInstance =
- new ArrayList<>(instances.size());
-
- for (ApplicationReport report : instances) {
- if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
- ||
- report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
- liveInstance.add(report);
- } else {
- nonLiveInstance.add(report);
- }
- }
-
- if (liveInstance.size() > 1) {
- Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
- }
- if (nonLiveInstance.size() > 1) {
- Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
- }
- instances.clear();
- instances.addAll(liveInstance);
- instances.addAll(nonLiveInstance);
- }
-
- /**
- * Merge in one map to another -all entries in the second map are
- * merged into the first -overwriting any duplicate keys.
- * @param first first map -the updated one.
- * @param second the map that is merged in
- * @return the first map
- */
- public static Map<String, String> mergeMap(Map<String, String> first,
- Map<String, String> second) {
- first.putAll(second);
- return first;
- }
-
- /**
- * Merge a set of entries into a map. This will take the entryset of
- * a map, or a Hadoop collection itself
- * @param dest destination
- * @param entries entries
- * @return dest -with the entries merged in
- */
- public static Map<String, String> mergeEntries(Map<String, String> dest,
- Iterable<Map.Entry<String, String>> entries) {
- for (Map.Entry<String, String> entry : entries) {
- dest.put(entry.getKey(), entry.getValue());
- }
- return dest;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
- Map<T1, T2> second) {
- first.putAll(second);
- return first;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
- Map<T1, T2> second) {
- Preconditions.checkArgument(first != null, "Null 'first' value");
- Preconditions.checkArgument(second != null, "Null 'second' value");
- for (Map.Entry<T1, T2> entry : second.entrySet()) {
- T1 key = entry.getKey();
- if (!first.containsKey(key)) {
- first.put(key, entry.getValue());
- }
- }
- return first;
- }
-
- /**
- * Convert a map to a multi-line string for printing
- * @param map map to stringify
- * @return a string representation of the map
- */
- public static String stringifyMap(Map<String, String> map) {
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, String> entry : map.entrySet()) {
- builder.append(entry.getKey())
- .append("=\"")
- .append(entry.getValue())
- .append("\"\n");
-
- }
- return builder.toString();
- }
-
- /**
- * Parse an int value, replacing it with defval if undefined;
- * @param errorKey key to use in exceptions
- * @param defVal default value to use if the key is not in the map
- * @param min min value or -1 for do not check
- * @param max max value or -1 for do not check
- * @return the int value the integer value
- * @throws BadConfigException if the value could not be parsed
- */
- public static int parseAndValidate(String errorKey,
- String valS,
- int defVal,
- int min, int max) throws
- BadConfigException {
- if (valS == null) {
- valS = Integer.toString(defVal);
- }
- String trim = valS.trim();
- int val;
- try {
- val = Integer.decode(trim);
- } catch (NumberFormatException e) {
- throw new BadConfigException("Failed to parse value of "
- + errorKey + ": \"" + trim + "\"");
- }
- if (min >= 0 && val < min) {
- throw new BadConfigException("Value of "
- + errorKey + ": " + val + ""
- + "is less than the minimum of " + min);
- }
- if (max >= 0 && val > max) {
- throw new BadConfigException("Value of "
- + errorKey + ": " + val + ""
- + "is more than the maximum of " + max);
- }
- return val;
- }
-
- public static InetSocketAddress getRmAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
- }
-
- public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
- }
-
- /**
- * probe to see if the RM scheduler is defined
- * @param conf config
- * @return true if the RM scheduler address is set to
- * something other than 0.0.0.0
- */
- public static boolean isRmSchedulerAddressDefined(Configuration conf) {
- InetSocketAddress address = getRmSchedulerAddress(conf);
- return isAddressDefined(address);
- }
-
- /**
- * probe to see if the address
- * @param address network address
- * @return true if the scheduler address is set to
- * something other than 0.0.0.0
- */
- public static boolean isAddressDefined(InetSocketAddress address) {
- if (address == null || address.getHostString() == null) {
- return false;
- }
- return !(address.getHostString().equals("0.0.0.0"));
- }
-
- public static void setRmAddress(Configuration conf, String rmAddr) {
- conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
- }
-
- public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
- conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
- }
-
- public static boolean hasAppFinished(ApplicationReport report) {
- return report == null ||
- report.getYarnApplicationState().ordinal() >=
- YarnApplicationState.FINISHED.ordinal();
- }
-
- public static String containerToString(Container container) {
- if (container == null) {
- return "null container";
- }
- return String.format(Locale.ENGLISH,
- "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
- container.getId(),
- container.getNodeId(),
- container.getNodeHttpAddress(),
- container.getPriority(),
- container.getResource());
- }
-
- /**
- * convert an AM report to a string for diagnostics
- * @param report the report
- * @return the string value
- */
- public static String reportToString(ApplicationReport report) {
- if (report == null) {
- return "Null application report";
- }
-
- return "App " + report.getName() + "/" + report.getApplicationType() +
- "# " +
- report.getApplicationId() + " user " + report.getUser() +
- " is in state " + report.getYarnApplicationState() +
- " RPC: " + report.getHost() + ":" + report.getRpcPort() +
- " URL: " + report.getOriginalTrackingUrl();
- }
-
- /**
- * Convert a YARN URL into a string value of a normal URL
- * @param url URL
- * @return string representatin
- */
- public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
- StringBuilder builder = new StringBuilder();
- builder.append(url.getScheme()).append("://");
- if (url.getHost() != null) {
- builder.append(url.getHost()).append(":").append(url.getPort());
- }
- builder.append(url.getFile());
- return builder.toString();
- }
-
- /**
- * Get a random open port
- * @return true if the port was available for listening on
- */
- public static int getOpenPort() throws IOException {
- ServerSocket socket = null;
- try {
- socket = new ServerSocket(0);
- return socket.getLocalPort();
- } finally {
- if (socket != null) {
- socket.close();
- }
- }
- }
-
- /**
- * See if a port is available for listening on by trying to listen
- * on it and seeing if that works or fails.
- * @param port port to listen to
- * @return true if the port was available for listening on
- */
- public static boolean isPortAvailable(int port) {
- try {
- ServerSocket socket = new ServerSocket(port);
- socket.close();
- return true;
- } catch (IOException e) {
- return false;
- }
- }
-
- // Build env map: key -> value;
- // value will be replaced by the corresponding value in tokenMap, if any.
- public static Map<String, String> buildEnvMap(
- org.apache.hadoop.yarn.service.api.records.Configuration conf,
- Map<String,String> tokenMap) {
- if (tokenMap == null) {
- return conf.getEnv();
- }
- Map<String, String> env = new HashMap<>();
- for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- for (Map.Entry<String,String> token : tokenMap.entrySet()) {
- val = val.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- }
- env.put(key,val);
- }
- return env;
- }
-
- /**
- * Apply a set of command line options to a cluster role map
- * @param clusterRoleMap cluster role map to merge onto
- * @param commandOptions command opts
- */
- public static void applyCommandLineRoleOptsToRoleMap(
- Map<String, Map<String, String>> clusterRoleMap,
- Map<String, Map<String, String>> commandOptions) {
- for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
- String key = entry.getKey();
- Map<String, String> optionMap = entry.getValue();
- Map<String, String> existingMap = clusterRoleMap.get(key);
- if (existingMap == null) {
- existingMap = new HashMap<String, String>();
- }
- log.debug("Overwriting role options with command line values {}",
- stringifyMap(optionMap));
- mergeMap(existingMap, optionMap);
- //set or overwrite the role
- clusterRoleMap.put(key, existingMap);
- }
- }
-
- /**
- * verify that the supplied cluster name is valid
- * @param clustername cluster name
- * @throws BadCommandArgumentsException if it is invalid
- */
- public static void validateClusterName(String clustername)
- throws BadCommandArgumentsException {
- if (!isClusternameValid(clustername)) {
- throw new BadCommandArgumentsException(
- "Illegal cluster name: " + clustername);
- }
- }
-
- /**
- * Verify that a Kerberos principal has been set -if not fail
- * with an error message that actually tells you what is missing
- * @param conf configuration to look at
- * @param principal key of principal
- * @throws BadConfigException if the key is not set
- */
- public static void verifyPrincipalSet(Configuration conf,
- String principal) throws
- BadConfigException {
- String principalName = conf.get(principal);
- if (principalName == null) {
- throw new BadConfigException("Unset Kerberos principal : %s",
- principal);
- }
- log.debug("Kerberos princial {}={}", principal, principalName);
- }
-
- /**
- * Flag to indicate whether the cluster is in secure mode
- * @param conf configuration to look at
- * @return true if the slider client/service should be in secure mode
- */
- public static boolean isHadoopClusterSecure(Configuration conf) {
- return SecurityUtil.getAuthenticationMethod(conf) !=
- UserGroupInformation.AuthenticationMethod.SIMPLE;
- }
-
- /**
- * Init security if the cluster configuration declares the cluster is secure
- * @param conf configuration to look at
- * @return true if the cluster is secure
- * @throws IOException cluster is secure
- * @throws SliderException the configuration/process is invalid
- */
- public static boolean maybeInitSecurity(Configuration conf) throws
- IOException,
- SliderException {
- boolean clusterSecure = isHadoopClusterSecure(conf);
- if (clusterSecure) {
- log.debug("Enabling security");
- initProcessSecurity(conf);
- }
- return clusterSecure;
- }
-
- /**
- * Turn on security. This is setup to only run once.
- * @param conf configuration to build up security
- * @return true if security was initialized in this call
- * @throws IOException IO/Net problems
- * @throws BadConfigException the configuration and system state are inconsistent
- */
- public static boolean initProcessSecurity(Configuration conf) throws
- IOException,
- SliderException {
-
- if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
- //security is already inited
- return false;
- }
-
- log.info("JVM initialized into secure mode with kerberos realm {}",
- SliderUtils.getKerberosRealm());
- //this gets UGI to reset its previous world view (i.e simple auth)
- //security
- log.debug("java.security.krb5.realm={}",
- System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
- log.debug("java.security.krb5.kdc={}",
- System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
- log.debug("hadoop.security.authentication={}",
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
- log.debug("hadoop.security.authorization={}",
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
- UserGroupInformation.setConfiguration(conf);
- UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
- log.debug("Authenticating as {}", authUser);
- log.debug("Login user is {}", UserGroupInformation.getLoginUser());
- if (!UserGroupInformation.isSecurityEnabled()) {
- throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED,
- "Although secure mode is enabled," +
- "the application has already set up its user as an insecure entity %s",
- authUser);
- }
- if (authUser.getAuthenticationMethod() ==
- UserGroupInformation.AuthenticationMethod.SIMPLE) {
- throw new BadConfigException("Auth User is not Kerberized %s" +
- " -security has already been set up with the wrong authentication method. "
- + "This can occur if a file system has already been created prior to the loading of "
- + "the security configuration.",
- authUser);
-
- }
-
- SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal");
- return true;
- }
-
- /**
- * Force an early login: This catches any auth problems early rather than
- * in RPC operations
- * @throws IOException if the login fails
- */
- public static void forceLogin() throws IOException {
- if (UserGroupInformation.isSecurityEnabled()) {
- if (UserGroupInformation.isLoginKeytabBased()) {
- UserGroupInformation.getLoginUser().reloginFromKeytab();
- } else {
- UserGroupInformation.getLoginUser().reloginFromTicketCache();
- }
- }
- }
-
- public static String getLibDir() {
- String[] libDirs = getLibDirs();
- if (libDirs == null || libDirs.length == 0) {
- return null;
- }
- return libDirs[0];
- }
-
- public static String[] getLibDirs() {
- String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
- if (isUnset(libDirStr)) {
- return ArrayUtils.EMPTY_STRING_ARRAY;
- }
- return StringUtils.split(libDirStr, ',');
- }
-
- /**
- * Submit a JAR containing a specific class and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param clazz class to look for
- * @param libdir lib directory
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public static LocalResource putJar(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Class clazz,
- Path tempPath,
- String libdir,
- String jarName
- )
- throws IOException, SliderException {
- LocalResource res = sliderFileSystem.submitJarWithClass(
- clazz,
- tempPath,
- libdir,
- jarName);
- providerResources.put(libdir + "/" + jarName, res);
- return res;
- }
-
- /**
- * Submit a JAR containing and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param libDir lib directory
- * @param srcPath copy jars from
- */
- public static void putAllJars(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String srcPath) throws IOException, SliderException {
- log.info("Loading all dependencies from {}", srcPath);
- if (SliderUtils.isSet(srcPath)) {
- File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = createJarFilter();
- File[] listOfJars = srcFolder.listFiles(jarFilter);
- if (listOfJars == null || listOfJars.length == 0) {
- return;
- }
- for (File jarFile : listOfJars) {
- LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
- providerResources.put(libDir + "/" + jarFile.getName(), res);
- }
- }
- }
-
- /**
- * Accept all filenames ending with {@code .jar}
- * @return a filename filter
- */
- public static FilenameFilter createJarFilter() {
- return new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
- }
- };
- }
-
- /**
- * Submit the AM tar.gz containing all dependencies and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- */
- public static void putAmTarGzipAndUpdate(
- Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem
- ) throws IOException, SliderException {
- log.info("Loading all dependencies from {}{}",
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
- sliderFileSystem.submitTarGzipAndUpdate(providerResources);
- }
-
- public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
- Map<String, Map<String, String>> dest = new HashMap<>();
- for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
- dest.put(entry.getKey(), stringMapClone(entry.getValue()));
- }
- return dest;
- }
-
- public static Map<String, String> stringMapClone(Map<String, String> src) {
- Map<String, String> dest = new HashMap<>();
- return mergeEntries(dest, src.entrySet());
- }
-
- /**
- * List a directory in the local filesystem
- * @param dir directory
- * @return a listing, one to a line
- */
- public static String listDir(File dir) {
- if (dir == null) {
- return "";
- }
- String[] confDirEntries = dir.list();
- if (confDirEntries == null) {
- return "";
- }
- StringBuilder builder = new StringBuilder();
- for (String entry : confDirEntries) {
- builder.append(entry).append("\n");
- }
- return builder.toString();
- }
-
- /**
- * Create a file:// path from a local file
- * @param file file to point the path
- * @return a new Path
- */
- public static Path createLocalPath(File file) {
- return new Path(file.toURI());
- }
-
- public static String getKerberosRealm() {
- try {
- return KerberosUtil.getDefaultRealm();
- } catch (Exception e) {
- log.debug("introspection into JVM internals failed", e);
- return "(unknown)";
-
- }
- }
-
- /**
- * Build up the classpath for execution
- * -behaves very differently on a mini test cluster vs a production
- * production one.
- *
- * @param sliderConfDir relative path to the dir containing slider config
- * options to put on the classpath -or null
- * @param libdir directory containing the JAR files
- * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
- * (and hence the current classpath should be used, not anything built up)
- * @return a classpath
- */
- public static ClasspathConstructor buildClasspath(String sliderConfDir,
- String libdir,
- SliderFileSystem sliderFileSystem,
- boolean usingMiniMRCluster) {
-
- ClasspathConstructor classpath = new ClasspathConstructor();
- classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
-
- // add the runtime classpath needed for tests to work
- if (usingMiniMRCluster) {
- // for mini cluster we pass down the java CP properties
- // and nothing else
- classpath.appendAll(classpath.localJVMClasspath());
- } else {
- if (sliderConfDir != null) {
- classpath.addClassDirectory(sliderConfDir);
- }
- classpath.addLibDir(libdir);
- if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
- classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
- } else {
- log.info(
- "For faster submission of apps, upload dependencies using cmd {} {}",
- SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
- }
- classpath.addRemoteClasspathEnvVar();
- classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
- }
- return classpath;
- }
-
- /**
- * Verify that a path refers to a directory. If not
- * logs the parent dir then throws an exception
- * @param dir the directory
- * @param errorlog log for output on an error
- * @throws FileNotFoundException if it is not a directory
- */
- public static void verifyIsDir(File dir, Logger errorlog) throws
- FileNotFoundException {
- if (!dir.exists()) {
- errorlog.warn("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
- throw new FileNotFoundException(dir.toString());
- }
- if (!dir.isDirectory()) {
- errorlog.info("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
- throw new FileNotFoundException(
- "Not a directory: " + dir);
- }
- }
-
- /**
- * Verify that a file exists
- * @param file file
- * @param errorlog log for output on an error
- * @throws FileNotFoundException
- */
- public static void verifyFileExists(File file, Logger errorlog) throws
- FileNotFoundException {
- if (!file.exists()) {
- errorlog.warn("contents of {}: {}", file,
- listDir(file.getParentFile()));
- throw new FileNotFoundException(file.toString());
- }
- if (!file.isFile()) {
- throw new FileNotFoundException("Not a file: " + file.toString());
- }
- }
-
- /**
- * verify that a config option is set
- * @param configuration config
- * @param key key
- * @return the value, in case it needs to be verified too
- * @throws BadConfigException if the key is missing
- */
- public static String verifyOptionSet(Configuration configuration, String key,
- boolean allowEmpty) throws BadConfigException {
- String val = configuration.get(key);
- if (val == null) {
- throw new BadConfigException(
- "Required configuration option \"%s\" not defined ", key);
- }
- if (!allowEmpty && val.isEmpty()) {
- throw new BadConfigException(
- "Configuration option \"%s\" must not be empty", key);
- }
- return val;
- }
-
- /**
- * Verify that a keytab property is defined and refers to a non-empty file
- *
- * @param siteConf configuration
- * @param prop property to look for
- * @return the file referenced
- * @throws BadConfigException on a failure
- */
- public static File verifyKeytabExists(Configuration siteConf,
- String prop) throws
- BadConfigException {
- String keytab = siteConf.get(prop);
- if (keytab == null) {
- throw new BadConfigException("Missing keytab property %s",
- prop);
-
- }
- File keytabFile = new File(keytab);
- if (!keytabFile.exists()) {
- throw new BadConfigException("Missing keytab file %s defined in %s",
- keytabFile,
- prop);
- }
- if (keytabFile.length() == 0 || !keytabFile.isFile()) {
- throw new BadConfigException("Invalid keytab file %s defined in %s",
- keytabFile,
- prop);
- }
- return keytabFile;
- }
-
- /**
- * Add a subpath to an existing URL. This extends
- * the path, inserting a / between all entries
- * if needed.
- * @param base base path/URL
- * @param path subpath
- * @return base+"/"+subpath
- */
- public static String appendToURL(String base, String path) {
- StringBuilder fullpath = new StringBuilder(base);
- if (!base.endsWith("/")) {
- fullpath.append("/");
- }
- if (path.startsWith("/")) {
- fullpath.append(path.substring(1));
- } else {
- fullpath.append(path);
- }
- return fullpath.toString();
- }
-
- /**
- * Truncate the given string to a maximum length provided
- * with a pad (...) added to the end if expected size if more than 10.
- * @param toTruncate string to truncate; may be null
- * @param maxSize maximum size
- * @return the truncated/padded string.
- */
- public static String truncate(String toTruncate, int maxSize) {
- if (toTruncate == null || maxSize < 1
- || toTruncate.length() <= maxSize) {
- return toTruncate;
- }
-
- String pad = "...";
- if (maxSize < 10) {
- pad = "";
- }
- return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
- }
-
- /**
- * Given a source folder create zipped file
- *
- * @param srcFolder
- * @param zipFile
- *
- * @throws IOException
- */
- public static void zipFolder(File srcFolder, File zipFile) throws IOException {
- log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true);
-
- byte[] buffer = new byte[1024];
-
- try (FileOutputStream fos = new FileOutputStream(zipFile)) {
- try (ZipOutputStream zos = new ZipOutputStream(fos)) {
-
- for (String file : files) {
- ZipEntry ze = new ZipEntry(file);
- zos.putNextEntry(ze);
- try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
- int len;
- while ((len = in.read(buffer)) > 0) {
- zos.write(buffer, 0, len);
- }
- }
- }
- }
- }
- }
-
- /**
- * Given a source folder create a tar.gz file
- *
- * @param libDirs
- * @param tarGzipFile
- *
- * @throws IOException
- */
- public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
- FilenameFilter filter) throws IOException {
- log.info("Tar-gzipping folders {} to {}", libDirs,
- tarGzipFile.getAbsolutePath());
-
- try(TarArchiveOutputStream taos =
- new TarArchiveOutputStream(new GZIPOutputStream(
- new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
- for (String libDir : libDirs) {
- File srcFolder = new File(libDir);
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true, filter);
- for (String file : files) {
- File srcFile = new File(srcFolder, file);
- TarArchiveEntry tarEntry = new TarArchiveEntry(
- srcFile, file);
- taos.putArchiveEntry(tarEntry);
- try(FileInputStream in = new FileInputStream(srcFile)) {
- org.apache.commons.io.IOUtils.copy(in, taos);
- }
- taos.flush();
- taos.closeArchiveEntry();
- }
- }
- }
- }
-
- private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative) {
- generateFileList(fileList, node, rootFolder, relative, null);
- }
-
- private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative, FilenameFilter filter) {
- if (node.isFile()) {
- String fileFullPath = node.toString();
- if (relative) {
- fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
- fileFullPath.length()));
- } else {
- fileList.add(fileFullPath);
- }
- }
-
- if (node.isDirectory()) {
- String[] subNode = node.list(filter);
- if (subNode == null || subNode.length == 0) {
- return;
- }
- for (String filename : subNode) {
- generateFileList(fileList, new File(node, filename), rootFolder,
- relative, filter);
- }
- }
- }
-
- /**
- * Check for any needed libraries being present. On Unix none are needed;
- * on windows they must be present
- * @return true if all is well
- */
- public static String checkForRequiredNativeLibraries() {
-
- if (!Shell.WINDOWS) {
- return "";
- }
- StringBuilder errorText = new StringBuilder("");
- if (!NativeIO.isAvailable()) {
- errorText.append("No native IO library. ");
- }
- try {
- String path = Shell.getQualifiedBinPath(WINUTILS);
- log.debug("winutils is at {}", path);
- } catch (IOException e) {
- errorText.append("No " + WINUTILS);
- log.warn("No winutils: {}", e, e);
- }
- try {
- File target = new File("target");
- FileUtil.canRead(target);
- } catch (UnsatisfiedLinkError e) {
- log.warn("Failing to link to native IO methods: {}", e, e);
- errorText.append("No native IO methods");
- }
- return errorText.toString();
- }
-
- /**
- * Strictly verify that windows utils is present.
- * Checks go as far as opening the file and looking for
- * the headers.
- * @throws IOException on any problem reading the file
- * @throws FileNotFoundException if the file is not considered valid
- */
- public static void maybeVerifyWinUtilsValid() throws
- IOException,
- SliderException {
- String errorText = SliderUtils.checkForRequiredNativeLibraries();
- if (!errorText.isEmpty()) {
- throw new BadClusterStateException(errorText);
- }
- }
-
- /**
- * Write bytes to a file
- * @param outfile output file
- * @param data data to write
- * @throws IOException on any IO problem
- */
- public static void write(File outfile, byte[] data)
- throws IOException {
- File parentDir = outfile.getCanonicalFile().getParentFile();
- if (parentDir == null) {
- throw new IOException(outfile.getPath() + " has no parent dir");
- }
- if (!parentDir.exists()) {
- if(!parentDir.mkdirs()) {
- throw new IOException("Failed to create parent directory " + parentDir);
- }
- }
- SliderUtils.verifyIsDir(parentDir, log);
- try(FileOutputStream out = new FileOutputStream(outfile)) {
- out.write(data);
- }
- }
-
- /**
- * Compare the times of two applications: most recent app comes first
- * Specifically: the one whose start time value is greater.
- */
- private static class MostRecentlyStartedAppFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getStartTime();
- long y = r2.getStartTime();
- return compareTwoLongsReverse(x, y);
- }
- }
-
- /**
- * Compare the times of two applications: most recent app comes first.
- * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
- */
- private static class MostRecentlyStartedOrFinishedFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long started1 = r1.getStartTime();
- long started2 = r2.getStartTime();
- long finished1 = r1.getFinishTime();
- long finished2 = r2.getFinishTime();
- long lastEvent1 = Math.max(started1, finished1);
- long lastEvent2 = Math.max(started2, finished2);
- return compareTwoLongsReverse(lastEvent1, lastEvent2);
- }
- }
-
- /**
- * Compare the times of two applications: most recently finished app comes first
- * Specifically: the one whose finish time value is greater.
- */
- private static class MostRecentAppFinishFirst
- implements Comparator<ApplicationReport>, Serializable {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getFinishTime();
- long y = r2.getFinishTime();
- return compareTwoLongsReverse(x, y);
- }
- }
-
- /**
- * Compare two long values for sorting. As the return value for
- * comparators must be int, the simple value of <code>x-y</code>
- * is inapplicable
- * @param x x value
- * @param y y value
- * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
- */
- public static int compareTwoLongsReverse(long x, long y) {
- return (x < y) ? 1 : ((x == y) ? 0 : -1);
- }
-
- public static String createNameTag(String name) {
- return "Name: " + name;
- }
-
- public static String createVersionTag(String version) {
- return "Version: " + version;
- }
-
- public static String createDescriptionTag(String description) {
- return "Description: " + description;
- }
-
- public static final String DAYS = ".days";
- public static final String HOURS = ".hours";
- public static final String MINUTES = ".minutes";
- public static final String SECONDS = ".seconds";
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
deleted file mode 100644
index e10305a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import java.util.Locale;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public enum ConfigFormat {
-
- JSON("json"),
- PROPERTIES("properties"),
- XML("xml"),
- HADOOP_XML("hadoop_xml"),
- ENV("env"),
- TEMPLATE("template"),
- YAML("yaml"),
- ;
- ConfigFormat(String suffix) {
- this.suffix = suffix;
- }
-
- private final String suffix;
-
- public String getSuffix() {
- return suffix;
- }
-
-
- @Override
- public String toString() {
- return suffix;
- }
-
- /**
- * Get a matching format or null
- * @param type
- * @return the format
- */
- public static ConfigFormat resolve(String type) {
- for (ConfigFormat format: values()) {
- if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
- return format;
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
deleted file mode 100644
index 0ac508b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Set of configuration properties that can be injected into the application
- * components via envs, files and custom pluggable helper docker containers.
- * Files of several standard formats like xml, properties, json, yaml and
- * templates will be supported.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Configuration implements Serializable {
- private static final long serialVersionUID = -4330788704981074466L;
-
- private Map<String, String> properties = new HashMap<String, String>();
- private Map<String, String> env = new HashMap<String, String>();
- private List<ConfigFile> files = new ArrayList<ConfigFile>();
-
- /**
- * A blob of key-value pairs of common application properties.
- **/
- public Configuration properties(Map<String, String> properties) {
- this.properties = properties;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common application properties.")
- @JsonProperty("properties")
- public Map<String, String> getProperties() {
- return properties;
- }
-
- public void setProperties(Map<String, String> properties) {
- this.properties = properties;
- }
-
- /**
- * A blob of key-value pairs which will be appended to the default system
- * properties and handed off to the application at start time. All placeholder
- * references to properties will be substituted before injection.
- **/
- public Configuration env(Map<String, String> env) {
- this.env = env;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.")
- @JsonProperty("env")
- public Map<String, String> getEnv() {
- return env;
- }
-
- public void setEnv(Map<String, String> env) {
- this.env = env;
- }
-
- /**
- * Array of list of files that needs to be created and made available as
- * volumes in the application component containers.
- **/
- public Configuration files(List<ConfigFile> files) {
- this.files = files;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the application component containers.")
- @JsonProperty("files")
- public List<ConfigFile> getFiles() {
- return files;
- }
-
- public void setFiles(List<ConfigFile> files) {
- this.files = files;
- }
-
- public long getPropertyLong(String name, long defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return Long.parseLong(value);
- }
-
- public int getPropertyInt(String name, int defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return Integer.parseInt(value);
- }
-
- public boolean getPropertyBool(String name, boolean defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return Boolean.parseBoolean(value);
- }
-
- public String getProperty(String name, String defaultValue) {
- String value = getProperty(name);
- if (StringUtils.isEmpty(value)) {
- return defaultValue;
- }
- return value;
- }
-
- public void setProperty(String name, String value) {
- properties.put(name, value);
- }
-
- public String getProperty(String name) {
- return properties.get(name.trim());
- }
-
- public String getEnv(String name) {
- return env.get(name.trim());
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Configuration configuration = (Configuration) o;
- return Objects.equals(this.properties, configuration.properties)
- && Objects.equals(this.env, configuration.env)
- && Objects.equals(this.files, configuration.files);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(properties, env, files);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Configuration {\n");
-
- sb.append(" properties: ").append(toIndentedString(properties))
- .append("\n");
- sb.append(" env: ").append(toIndentedString(env)).append("\n");
- sb.append(" files: ").append(toIndentedString(files)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-
- /**
- * Merge all properties and envs from that configuration to this configration.
- * For ConfigFiles, all properties and envs of that ConfigFile are merged into
- * this ConfigFile.
- */
- public synchronized void mergeFrom(Configuration that) {
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
- .getProperties());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
-
- Map<String, ConfigFile> thatMap = new HashMap<>();
- for (ConfigFile file : that.getFiles()) {
- thatMap.put(file.getDestFile(), file.copy());
- }
- for (ConfigFile thisFile : files) {
- if(thatMap.containsKey(thisFile.getDestFile())) {
- ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
- thatFile.getProps());
- thatMap.remove(thisFile.getDestFile());
- }
- }
- // add remaining new files from that Configration
- for (ConfigFile thatFile : thatMap.values()) {
- files.add(thatFile.copy());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
deleted file mode 100644
index 8b687bb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Date;
-import java.util.Objects;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * An instance of a running application container.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "An instance of a running application container")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Container extends BaseResource {
- private static final long serialVersionUID = -8955788064529288L;
-
- private String id = null;
- private Date launchTime = null;
- private String ip = null;
- private String hostname = null;
- private String bareHost = null;
- private ContainerState state = null;
- private String componentName = null;
- private Resource resource = null;
- private Artifact artifact = null;
- private Boolean privilegedContainer = null;
-
- /**
- * Unique container id of a running application, e.g.
- * container_e3751_1458061340047_0008_01_000002.
- **/
- public Container id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z.
- * This will most likely be different from cluster launch time.
- **/
- public Container launchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
- @JsonProperty("launch_time")
- public Date getLaunchTime() {
- return launchTime == null ? null : (Date) launchTime.clone();
- }
-
- @XmlElement(name = "launch_time")
- public void setLaunchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- }
-
- /**
- * IP address of a running container, e.g. 172.31.42.141. The IP address and
- * hostname attribute values are dependent on the cluster/docker network setup
- * as per YARN-4007.
- **/
- public Container ip(String ip) {
- this.ip = ip;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
- @JsonProperty("ip")
- public String getIp() {
- return ip;
- }
-
- public void setIp(String ip) {
- this.ip = ip;
- }
-
- /**
- * Fully qualified hostname of a running container, e.g.
- * ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and
- * hostname attribute values are dependent on the cluster/docker network setup
- * as per YARN-4007.
- **/
- public Container hostname(String hostname) {
- this.hostname = hostname;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
- @JsonProperty("hostname")
- public String getHostname() {
- return hostname;
- }
-
- public void setHostname(String hostname) {
- this.hostname = hostname;
- }
-
- /**
- * The bare node or host in which the container is running, e.g.
- * cn008.example.com.
- **/
- public Container bareHost(String bareHost) {
- this.bareHost = bareHost;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The bare node or host in which the container is running, e.g. cn008.example.com.")
- @JsonProperty("bare_host")
- public String getBareHost() {
- return bareHost;
- }
-
- @XmlElement(name = "bare_host")
- public void setBareHost(String bareHost) {
- this.bareHost = bareHost;
- }
-
- /**
- * State of the container of an application.
- **/
- public Container state(ContainerState state) {
- this.state = state;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "State of the container of an application.")
- @JsonProperty("state")
- public ContainerState getState() {
- return state;
- }
-
- public void setState(ContainerState state) {
- this.state = state;
- }
-
- /**
- * Name of the component that this container instance belongs to.
- **/
- public Container componentName(String componentName) {
- this.componentName = componentName;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Name of the component that this container instance belongs to.")
- @JsonProperty("component_name")
- public String getComponentName() {
- return componentName;
- }
-
- @XmlElement(name = "component_name")
- public void setComponentName(String componentName) {
- this.componentName = componentName;
- }
-
- /**
- * Resource used for this container.
- **/
- public Container resource(Resource resource) {
- this.resource = resource;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Resource used for this container.")
- @JsonProperty("resource")
- public Resource getResource() {
- return resource;
- }
-
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- /**
- * Artifact used for this container.
- **/
- public Container artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact used for this container.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- /**
- * Container running in privileged mode or not.
- **/
- public Container privilegedContainer(Boolean privilegedContainer) {
- this.privilegedContainer = privilegedContainer;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Container running in privileged mode or not.")
- @JsonProperty("privileged_container")
- public Boolean getPrivilegedContainer() {
- return privilegedContainer;
- }
-
- public void setPrivilegedContainer(Boolean privilegedContainer) {
- this.privilegedContainer = privilegedContainer;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Container container = (Container) o;
- return Objects.equals(this.id, container.id);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(id);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Container {\n");
-
- sb.append(" id: ").append(toIndentedString(id)).append("\n");
- sb.append(" launchTime: ").append(toIndentedString(launchTime))
- .append("\n");
- sb.append(" ip: ").append(toIndentedString(ip)).append("\n");
- sb.append(" hostname: ").append(toIndentedString(hostname)).append("\n");
- sb.append(" bareHost: ").append(toIndentedString(bareHost)).append("\n");
- sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" componentName: ").append(toIndentedString(componentName))
- .append("\n");
- sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append(" privilegedContainer: ")
- .append(toIndentedString(privilegedContainer)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
deleted file mode 100644
index bf09ff2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * The current state of the container of an application.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public enum ContainerState {
- RUNNING_BUT_UNREADY, READY, STOPPED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
deleted file mode 100644
index c64b1b5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class Error {
-
- private Integer code = null;
- private String message = null;
- private String fields = null;
-
- /**
- **/
- public Error code(Integer code) {
- this.code = code;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "")
- @JsonProperty("code")
- public Integer getCode() {
- return code;
- }
-
- public void setCode(Integer code) {
- this.code = code;
- }
-
- /**
- **/
- public Error message(String message) {
- this.message = message;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "")
- @JsonProperty("message")
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- }
-
- /**
- **/
- public Error fields(String fields) {
- this.fields = fields;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "")
- @JsonProperty("fields")
- public String getFields() {
- return fields;
- }
-
- public void setFields(String fields) {
- this.fields = fields;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Error error = (Error) o;
- return Objects.equals(this.code, error.code)
- && Objects.equals(this.message, error.message)
- && Objects.equals(this.fields, error.fields);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(code, message, fields);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Error {\n");
-
- sb.append(" code: ").append(toIndentedString(code)).append("\n");
- sb.append(" message: ").append(toIndentedString(message)).append("\n");
- sb.append(" fields: ").append(toIndentedString(fields)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
deleted file mode 100644
index 7d1b889..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Placement policy of an instance of an application. This feature is in the
- * works in YARN-4902.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "Placement policy of an instance of an application. This feature is in the works in YARN-4902.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class PlacementPolicy implements Serializable {
- private static final long serialVersionUID = 4341110649551172231L;
-
- private String label = null;
-
- /**
- * Assigns an app to a named partition of the cluster where the application
- * desires to run (optional). If not specified all apps are submitted to a
- * default label of the app owner. One or more labels can be setup for each
- * application owner account with required constraints like no-preemption,
- * sla-99999, preemption-ok, etc.
- **/
- public PlacementPolicy label(String label) {
- this.label = label;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.")
- @JsonProperty("label")
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- PlacementPolicy placementPolicy = (PlacementPolicy) o;
- return Objects.equals(this.label, placementPolicy.label);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(label);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class PlacementPolicy {\n");
-
- sb.append(" label: ").append(toIndentedString(label)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
deleted file mode 100644
index eadbb48..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * A custom command or a pluggable helper container to determine the readiness
- * of a container of a component. Readiness for every application is different.
- * Hence the need for a simple interface, with scope to support advanced
- * usecases.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class ReadinessCheck implements Serializable {
- private static final long serialVersionUID = -3836839816887186801L;
-
- public enum TypeEnum {
- HTTP("HTTP"),
- PORT("PORT");
-
- private String value;
-
- TypeEnum(String value) {
- this.value = value;
- }
-
- @Override
- @JsonValue
- public String toString() {
- return value;
- }
- }
-
- private TypeEnum type = null;
- private Map<String, String> props = new HashMap<String, String>();
- private Artifact artifact = null;
-
- /**
- * E.g. HTTP (YARN will perform a simple REST call at a regular interval and
- * expect a 204 No content).
- **/
- public ReadinessCheck type(TypeEnum type) {
- this.type = type;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
- @JsonProperty("type")
- public TypeEnum getType() {
- return type;
- }
-
- public void setType(TypeEnum type) {
- this.type = type;
- }
-
- public ReadinessCheck props(Map<String, String> props) {
- this.props = props;
- return this;
- }
-
- public ReadinessCheck putPropsItem(String key, String propsItem) {
- this.props.put(key, propsItem);
- return this;
- }
-
- /**
- * A blob of key value pairs that will be used to configure the check.
- * @return props
- **/
- @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be used to configure the check.")
- public Map<String, String> getProps() {
- return props;
- }
-
- public void setProps(Map<String, String> props) {
- this.props = props;
- }
-
- /**
- * Artifact of the pluggable readiness check helper container (optional). If
- * specified, this helper container typically hosts the http uri and
- * encapsulates the complex scripts required to perform actual container
- * readiness check. At the end it is expected to respond a 204 No content just
- * like the simplified use case. This pluggable framework benefits application
- * owners who can run applications without any packaging modifications. Note,
- * artifacts of type docker only is supported for now.
- **/
- public ReadinessCheck artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ReadinessCheck readinessCheck = (ReadinessCheck) o;
- return Objects.equals(this.type, readinessCheck.type) &&
- Objects.equals(this.props, readinessCheck.props) &&
- Objects.equals(this.artifact, readinessCheck.artifact);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(type, props, artifact);
- }
-
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class ReadinessCheck {\n");
-
- sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
deleted file mode 100644
index bda79c9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Resource determines the amount of resources (vcores, memory, network, etc.)
- * usable by a container. This field determines the resource to be applied for
- * all the containers of a component or application. The resource specified at
- * the app (or global) level can be overriden at the component level. Only one
- * of profile OR cpu & memory are exepected. It raises a validation
- * exception otherwise.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class Resource extends BaseResource implements Cloneable {
- private static final long serialVersionUID = -6431667797380250037L;
-
- private String profile = null;
- private Integer cpus = 1;
- private String memory = null;
-
- /**
- * Each resource profile has a unique id which is associated with a
- * cluster-level predefined memory, cpus, etc.
- **/
- public Resource profile(String profile) {
- this.profile = profile;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.")
- @JsonProperty("profile")
- public String getProfile() {
- return profile;
- }
-
- public void setProfile(String profile) {
- this.profile = profile;
- }
-
- /**
- * Amount of vcores allocated to each container (optional but overrides cpus
- * in profile if specified).
- **/
- public Resource cpus(Integer cpus) {
- this.cpus = cpus;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).")
- @JsonProperty("cpus")
- public Integer getCpus() {
- return cpus;
- }
-
- public void setCpus(Integer cpus) {
- this.cpus = cpus;
- }
-
- /**
- * Amount of memory allocated to each container (optional but overrides memory
- * in profile if specified). Currently accepts only an integer value and
- * default unit is in MB.
- **/
- public Resource memory(String memory) {
- this.memory = memory;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.")
- @JsonProperty("memory")
- public String getMemory() {
- return memory;
- }
-
- public void setMemory(String memory) {
- this.memory = memory;
- }
-
- public long getMemoryMB() {
- if (this.memory == null) {
- return 0;
- }
- return Long.valueOf(memory);
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Resource resource = (Resource) o;
- return Objects.equals(this.profile, resource.profile)
- && Objects.equals(this.cpus, resource.cpus)
- && Objects.equals(this.memory, resource.memory);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(profile, cpus, memory);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Resource {\n");
-
- sb.append(" profile: ").append(toIndentedString(profile)).append("\n");
- sb.append(" cpus: ").append(toIndentedString(cpus)).append("\n");
- sb.append(" memory: ").append(toIndentedString(memory)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
deleted file mode 100644
index 0749077..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ClientAMProxy.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.client.ServerProxy;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-
-import java.net.InetSocketAddress;
-
-public class ClientAMProxy extends ServerProxy{
-
- public static <T> T createProxy(final Configuration conf,
- final Class<T> protocol, final UserGroupInformation ugi,
- final YarnRPC rpc, final InetSocketAddress serverAddress) {
-
- RetryPolicy retryPolicy =
- createRetryPolicy(conf, YarnServiceConf.CLIENT_AM_RETRY_MAX_WAIT_MS,
- 15 * 60 * 1000, YarnServiceConf.CLIENT_AM_RETRY_MAX_INTERVAL_MS,
- 2 * 1000);
- Configuration confClone = new Configuration(conf);
- confClone.setInt(
- CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- confClone.setInt(CommonConfigurationKeysPublic.
- IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
- return createRetriableProxy(confClone, protocol, ugi, rpc, serverAddress,
- retryPolicy);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
deleted file mode 100644
index c7421ff..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.*;
-
-public class ServiceCLI {
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceClient.class);
- protected ServiceClient client;
-
- int exec(ClientArgs args) throws Throwable {
- if (StringUtils.isEmpty(args.getAction())) {
- System.out.println(args.usage());
- return -1;
- }
- switch (args.getAction()) {
- case ACTION_BUILD: // Upload app json onto hdfs
- client.actionBuild(args.getActionBuildArgs());
- break;
- case ACTION_START: // start the app with the pre-uploaded app json on hdfs
- client.actionStart(args.getClusterName());
- break;
- case ACTION_CREATE: // create == build + start
- client.actionCreate(args.getActionCreateArgs());
- break;
- case ACTION_STATUS:
- Application app = client.getStatus(args.getClusterName());
- System.out.println(app);
- break;
- case ACTION_FLEX:
- client.actionFlexByCLI(args);
- break;
- case ACTION_STOP:
- client.actionStop(args.getClusterName(), false);
- break;
- case ACTION_DESTROY: // Destroy can happen only if app is already stopped
- client.actionDestroy(args.getClusterName());
- break;
- case ACTION_DEPENDENCY: // upload dependency jars
- client.actionDependency(args.getActionDependencyArgs());
- break;
- case ACTION_UPDATE:
- client.updateLifetime(args.getClusterName(),
- args.getActionUpdateArgs().lifetime);
- break;
- case ACTION_HELP:
- LOG.info(args.usage());
- break;
- default:
- LOG.info("NOT IMPLEMENTED: " + args.getAction());
- LOG.info(args.usage());
- return -1;
- }
- return 0;
- }
-
- public ServiceCLI() {
- createServiceClient();
- }
-
- protected void createServiceClient() {
- client = new ServiceClient();
- client.init(new YarnConfiguration());
- client.start();
- }
-
- public static void main(String[] args) throws Throwable {
- ClientArgs clientArgs = new ClientArgs(args);
- try {
- clientArgs.parse();
- } catch (BadCommandArgumentsException e) {
- System.err.println(e.getMessage());
- System.exit(-1);
- }
- ServiceCLI cli = new ServiceCLI();
- int res = cli.exec(clientArgs);
- System.exit(res);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[56/58] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN
related params in the service REST API. Contributed by Jian He
Posted by ji...@apache.org.
YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6c4bd74
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6c4bd74
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6c4bd74
Branch: refs/heads/yarn-native-services
Commit: a6c4bd74b636d0715ffb4c1aca9ef4e49201aa02
Parents: a288b0a
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Nov 3 11:25:57 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:20 2017 -0800
----------------------------------------------------------------------
.../definition/YARN-Services-Examples.md | 8 +-
...RN-Simplified-V1-API-Layer-For-Services.yaml | 6 +-
.../examples/httpd-no-dns/httpd-no-dns.json | 6 +-
.../examples/httpd/httpd.json | 4 +-
.../hadoop/yarn/service/ServiceMaster.java | 6 +
.../hadoop/yarn/service/ServiceMetrics.java | 4 -
.../hadoop/yarn/service/ServiceScheduler.java | 15 --
.../yarn/service/api/ServiceApiConstants.java | 4 +-
.../yarn/service/api/records/ConfigFile.java | 32 ++--
.../yarn/service/api/records/Configuration.java | 4 +-
.../service/api/records/ReadinessCheck.java | 24 +--
.../yarn/service/client/ServiceClient.java | 61 ++++---
.../yarn/service/component/Component.java | 29 ++--
.../component/instance/ComponentInstance.java | 37 +++--
.../yarn/service/conf/YarnServiceConf.java | 14 +-
.../yarn/service/conf/YarnServiceConstants.java | 2 +
.../service/monitor/probe/MonitorUtils.java | 4 +-
.../provider/AbstractClientProvider.java | 17 +-
.../yarn/service/provider/ProviderUtils.java | 12 +-
.../timelineservice/ServiceTimelineEvent.java | 4 +-
.../ServiceTimelinePublisher.java | 48 +++---
.../yarn/service/conf/TestAppJsonResolve.java | 8 +-
.../TestServiceTimelinePublisher.java | 2 +-
.../service/conf/examples/app-override.json | 6 +-
.../markdown/yarn-service/Configurations.md | 165 +++++++++++++++++++
.../src/site/markdown/yarn-service/Examples.md | 4 +-
.../src/site/markdown/yarn-service/Overview.md | 3 +-
.../markdown/yarn-service/YarnServiceAPI.md | 14 +-
.../app/controllers/yarn-app/components.js | 2 +-
.../app/templates/components/metrics-table.hbs | 2 +-
.../templates/yarn-component-instance/info.hbs | 4 +-
.../templates/yarn-component-instances/info.hbs | 2 +-
32 files changed, 384 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
index 3cd3d48..00b21dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
@@ -176,14 +176,14 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
{
"type": "XML",
"dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
+ "properties": {
"fs.defaultFS": "${CLUSTER_FS_URI}"
}
},
{
"type": "XML",
"dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
+ "properties": {
"hbase.cluster.distributed": "true",
"hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
"hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
@@ -216,14 +216,14 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
{
"type": "XML",
"dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
+ "properties": {
"fs.defaultFS": "${CLUSTER_FS_URI}"
}
},
{
"type": "XML",
"dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
+ "properties": {
"hbase.cluster.distributed": "true",
"hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
"hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index cc76259..088b50c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -340,7 +340,7 @@ definitions:
enum:
- HTTP
- PORT
- props:
+ properties:
type: object
description: A blob of key value pairs that will be used to configure the check.
additionalProperties:
@@ -353,7 +353,7 @@ definitions:
properties:
properties:
type: object
- description: A blob of key-value pairs of common service properties.
+ description: A blob of key-value pairs for configuring the YARN service AM
additionalProperties:
type: string
env:
@@ -386,7 +386,7 @@ definitions:
src_file:
type: string
description: This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.
- props:
+ properties:
type: object
description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.
Container:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
index 6b35538..7bf2f68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
@@ -16,16 +16,16 @@
},
"readiness_check": {
"type": "HTTP",
- "props": {
+ "properties": {
"url": "http://${THIS_HOST}:8080"
}
},
"configuration": {
"files": [
{
- "type": "ENV",
+ "type": "TEMPLATE",
"dest_file": "/var/www/html/index.html",
- "props": {
+ "properties": {
"content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
index e63376d..8786214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
@@ -17,9 +17,9 @@
"configuration": {
"files": [
{
- "type": "ENV",
+ "type": "TEMPLATE",
"dest_file": "/var/www/html/index.html",
- "props": {
+ "properties": {
"content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index d58e32b..2d86a5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -78,6 +78,12 @@ public class ServiceMaster extends CompositeService {
fs.setAppDir(appDir);
loadApplicationJson(context, fs);
+ // Take yarn config from YarnFile and merge them into YarnConfiguration
+ for (Map.Entry<String, String> entry : context.service
+ .getConfiguration().getProperties().entrySet()) {
+ conf.set(entry.getKey(), entry.getValue());
+ }
+
ContainerId amContainerId = getAMContainerId();
ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
index 9fc886e..b5bbb7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
@@ -35,9 +35,6 @@ public class ServiceMetrics implements MetricsSource {
@Metric("containers requested")
public MutableGaugeInt containersRequested;
- @Metric("anti-affinity containers pending")
- public MutableGaugeInt pendingAAContainers;
-
@Metric("containers running")
public MutableGaugeInt containersRunning;
@@ -86,7 +83,6 @@ public class ServiceMetrics implements MetricsSource {
@Override public String toString() {
return "ServiceMetrics{"
+ "containersRequested=" + containersRequested.value()
- + ", pendingAAContainers=" + pendingAAContainers.value()
+ ", containersRunning=" + containersRunning.value()
+ ", containersDesired=" + containersDesired.value()
+ ", containersSucceeded=" + containersSucceeded.value()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 631f89e..a7b7e22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -397,10 +397,6 @@ public class ServiceScheduler extends CompositeService {
PersistencePolicies.APPLICATION);
serviceRecord.description = "YarnServiceMaster";
- // set any provided attributes
- setUserProvidedServiceRecordAttributes(service.getConfiguration(),
- serviceRecord);
-
executorService.submit(new Runnable() {
@Override public void run() {
try {
@@ -426,17 +422,6 @@ public class ServiceScheduler extends CompositeService {
}
}
- private void setUserProvidedServiceRecordAttributes(
- org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) {
- String prefix = "service.record.attribute";
- for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
- if (entry.getKey().startsWith(prefix)) {
- String key = entry.getKey().substring(prefix.length() + 1);
- record.set(key, entry.getValue().trim());
- }
- }
- }
-
private void createAllComponents() {
long allocateId = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
index a85191c..a4f2243 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
@@ -53,9 +53,9 @@ public interface ServiceApiConstants {
String CONTAINER_ID = $("CONTAINER_ID");
// Templates for component instance host/IP
- String COMPONENT_HOST = $("%s_HOST");
+ String COMPONENT_INSTANCE_HOST = $("%s_HOST");
- String COMPONENT_IP = $("%s_IP");
+ String COMPONENT_INSTANCE_IP = $("%s_IP");
// Constants for default cluster ZK
String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
index fb088e2..984e6f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -49,13 +49,13 @@ public class ConfigFile implements Serializable {
private static final long serialVersionUID = -7009402089417704612L;
/**
- * Config Type. XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported.
+ * Config Type. XML, JSON, YAML, TEMPLATE and HADOOP_XML are supported.
**/
@XmlType(name = "config_type")
@XmlEnum
public enum TypeEnum {
XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
- "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
+ "TEMPLATE"), HADOOP_XML("HADOOP_XML"),;
private String value;
@@ -73,15 +73,15 @@ public class ConfigFile implements Serializable {
private TypeEnum type = null;
private String destFile = null;
private String srcFile = null;
- private Map<String, String> props = new HashMap<>();
+ private Map<String, String> properties = new HashMap<>();
public ConfigFile copy() {
ConfigFile copy = new ConfigFile();
copy.setType(this.getType());
copy.setSrcFile(this.getSrcFile());
copy.setDestFile(this.getDestFile());
- if (this.getProps() != null && !this.getProps().isEmpty()) {
- copy.getProps().putAll(this.getProps());
+ if (this.getProperties() != null && !this.getProperties().isEmpty()) {
+ copy.getProperties().putAll(this.getProperties());
}
return copy;
}
@@ -156,28 +156,28 @@ public class ConfigFile implements Serializable {
in the dest_file and these properties will overwrite, if any, existing
properties in src_file or be added as new properties in src_file.
**/
- public ConfigFile props(Map<String, String> props) {
- this.props = props;
+ public ConfigFile properties(Map<String, String> properties) {
+ this.properties = properties;
return this;
}
@ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type."
+ " If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any,"
+ " existing properties in src_file or be added as new properties in src_file.")
- @JsonProperty("props")
- public Map<String, String> getProps() {
- return props;
+ @JsonProperty("properties")
+ public Map<String, String> getProperties() {
+ return properties;
}
- public void setProps(Map<String, String> props) {
- this.props = props;
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
}
public long getLong(String name, long defaultValue) {
if (name == null) {
return defaultValue;
}
- String value = props.get(name.trim());
+ String value = properties.get(name.trim());
return Long.parseLong(value);
}
@@ -185,7 +185,7 @@ public class ConfigFile implements Serializable {
if (name == null) {
return defaultValue;
}
- return Boolean.valueOf(props.get(name.trim()));
+ return Boolean.valueOf(properties.get(name.trim()));
}
@Override
@@ -204,7 +204,7 @@ public class ConfigFile implements Serializable {
@Override
public int hashCode() {
- return Objects.hash(type, destFile, srcFile, props);
+ return Objects.hash(type, destFile, srcFile, properties);
}
@Override
@@ -215,7 +215,7 @@ public class ConfigFile implements Serializable {
sb.append(" type: ").append(toIndentedString(type)).append("\n");
sb.append(" destFile: ").append(toIndentedString(destFile)).append("\n");
sb.append(" srcFile: ").append(toIndentedString(srcFile)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append(" properties: ").append(toIndentedString(properties)).append("\n");
sb.append("}");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
index 2f8ca96..dc6dff8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
@@ -212,8 +212,8 @@ public class Configuration implements Serializable {
for (ConfigFile thisFile : files) {
if(thatMap.containsKey(thisFile.getDestFile())) {
ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
- thatFile.getProps());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
+ thatFile.getProperties());
thatMap.remove(thisFile.getDestFile());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
index b25828f..0a3713c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -69,7 +69,7 @@ public class ReadinessCheck implements Serializable {
}
private TypeEnum type = null;
- private Map<String, String> props = new HashMap<String, String>();
+ private Map<String, String> properties = new HashMap<String, String>();
private Artifact artifact = null;
/**
@@ -91,27 +91,27 @@ public class ReadinessCheck implements Serializable {
this.type = type;
}
- public ReadinessCheck props(Map<String, String> props) {
- this.props = props;
+ public ReadinessCheck properties(Map<String, String> properties) {
+ this.properties = properties;
return this;
}
public ReadinessCheck putPropsItem(String key, String propsItem) {
- this.props.put(key, propsItem);
+ this.properties.put(key, propsItem);
return this;
}
/**
* A blob of key value pairs that will be used to configure the check.
- * @return props
+ * @return properties
**/
@ApiModelProperty(example = "null", value = "A blob of key value pairs that will be used to configure the check.")
- public Map<String, String> getProps() {
- return props;
+ public Map<String, String> getProperties() {
+ return properties;
}
- public void setProps(Map<String, String> props) {
- this.props = props;
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
}
/**
@@ -148,13 +148,13 @@ public class ReadinessCheck implements Serializable {
}
ReadinessCheck readinessCheck = (ReadinessCheck) o;
return Objects.equals(this.type, readinessCheck.type) &&
- Objects.equals(this.props, readinessCheck.props) &&
+ Objects.equals(this.properties, readinessCheck.properties) &&
Objects.equals(this.artifact, readinessCheck.artifact);
}
@Override
public int hashCode() {
- return Objects.hash(type, props, artifact);
+ return Objects.hash(type, properties, artifact);
}
@@ -164,7 +164,7 @@ public class ReadinessCheck implements Serializable {
sb.append("class ReadinessCheck {\n");
sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append(" properties: ").append(toIndentedString(properties)).append("\n");
sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
sb.append("}");
return sb.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 636b127..775a642 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -38,17 +38,8 @@ import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+
+import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
@@ -99,7 +90,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
@@ -553,8 +544,11 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime());
submissionContext.setApplicationTimeouts(appTimeout);
}
- submissionContext.setMaxAppAttempts(conf.getInt(
- YarnServiceConf.AM_RESTART_MAX, 2));
+ submissionContext.setMaxAppAttempts(YarnServiceConf
+ .getInt(YarnServiceConf.AM_RESTART_MAX, 20, app.getConfiguration(),
+ conf));
+
+ setLogAggregationContext(app, conf, submissionContext);
Map<String, LocalResource> localResources = new HashMap<>();
@@ -568,14 +562,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
if (LOG.isDebugEnabled()) {
printLocalResources(localResources);
}
- Map<String, String> env = addAMEnv(conf);
+ Map<String, String> env = addAMEnv();
// create AM CLI
- String cmdStr =
- buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j);
+ String cmdStr = buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j);
submissionContext.setResource(Resource.newInstance(YarnServiceConf
- .getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM,
- app.getConfiguration(), conf), 1));
+ .getLong(YarnServiceConf.AM_RESOURCE_MEM,
+ YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM, app.getConfiguration(),
+ conf), 1));
String queue = app.getQueue();
if (StringUtils.isEmpty(queue)) {
queue = conf.get(YARN_QUEUE, "default");
@@ -598,6 +592,33 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return submissionContext.getApplicationId();
}
+ private void setLogAggregationContext(Service app, Configuration conf,
+ ApplicationSubmissionContext submissionContext) {
+ LogAggregationContext context = Records.newRecord(LogAggregationContext
+ .class);
+ String finalLogInclude = YarnServiceConf.get
+ (FINAL_LOG_INCLUSION_PATTERN, null, app.getConfiguration(), conf);
+ if (!StringUtils.isEmpty(finalLogInclude)) {
+ context.setIncludePattern(finalLogInclude);
+ }
+ String finalLogExclude = YarnServiceConf.get
+ (FINAL_LOG_EXCLUSION_PATTERN, null, app.getConfiguration(), conf);
+ if (!StringUtils.isEmpty(finalLogExclude)) {
+ context.setExcludePattern(finalLogExclude);
+ }
+ String rollingLogInclude = YarnServiceConf.get
+ (ROLLING_LOG_INCLUSION_PATTERN, null, app.getConfiguration(), conf);
+ if (!StringUtils.isEmpty(rollingLogInclude)) {
+ context.setRolledLogsIncludePattern(rollingLogInclude);
+ }
+ String rollingLogExclude = YarnServiceConf.get
+ (ROLLING_LOG_EXCLUSION_PATTERN, null, app.getConfiguration(), conf);
+ if (!StringUtils.isEmpty(rollingLogExclude)) {
+ context.setRolledLogsExcludePattern(rollingLogExclude);
+ }
+ submissionContext.setLogAggregationContext(context);
+ }
+
private void printLocalResources(Map<String, LocalResource> map) {
LOG.debug("Added LocalResource for localization: ");
StringBuilder builder = new StringBuilder();
@@ -635,7 +656,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return cmdStr;
}
- private Map<String, String> addAMEnv(Configuration conf) throws IOException {
+ private Map<String, String> addAMEnv() throws IOException {
Map<String, String> env = new HashMap<>();
ClasspathConstructor classpath =
buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index a5dd39c..7667a53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -116,7 +116,7 @@ public class Component implements EventHandler<ComponentEvent> {
.addTransition(FLEXING, FLEXING, CONTAINER_COMPLETED,
new ContainerCompletedTransition())
// Flex while previous flex is still in progress
- .addTransition(FLEXING, EnumSet.of(FLEXING), FLEX,
+ .addTransition(FLEXING, EnumSet.of(FLEXING, STABLE), FLEX,
new FlexComponentTransition())
// container failed while stable
@@ -214,14 +214,12 @@ public class Component implements EventHandler<ComponentEvent> {
new ArrayList<>(component.getAllComponentInstances());
// sort in Most recent -> oldest order, destroy most recent ones.
- Collections.sort(list, Collections.reverseOrder());
+ list.sort(Collections.reverseOrder());
for (int i = 0; i < delta; i++) {
ComponentInstance instance = list.get(i);
// remove the instance
component.compInstances.remove(instance.getCompInstanceName());
component.pendingInstances.remove(instance);
- component.componentMetrics.containersFailed.incr();
- component.componentMetrics.containersRunning.decr();
// decrement id counter
component.instanceIdCounter.decrementAndGet();
instance.destroy();
@@ -282,7 +280,6 @@ public class Component implements EventHandler<ComponentEvent> {
component.compInstanceDispatcher.getEventHandler().handle(
new ComponentInstanceEvent(instance.getContainerId(),
START));
- component.incRunningContainers();
}
}
@@ -294,7 +291,6 @@ public class Component implements EventHandler<ComponentEvent> {
component.compInstanceDispatcher.getEventHandler().handle(
new ComponentInstanceEvent(event.getInstance().getContainerId(),
START));
- component.incRunningContainers();
return checkIfStable(component);
}
}
@@ -413,9 +409,6 @@ public class Component implements EventHandler<ComponentEvent> {
componentMetrics.containersFailed.incr();
scheduler.getServiceMetrics().containersFailed.incr();
- // dec running container
- decRunningContainers();
-
if (Apps.shouldCountTowardsNodeBlacklisting(status.getExitStatus())) {
String host = scheduler.getLiveInstances().get(status.getContainerId())
.getNodeId().getHost();
@@ -468,31 +461,33 @@ public class Component implements EventHandler<ComponentEvent> {
}
String ip = instance.getContainerStatus().getIPs().get(0);
String host = instance.getContainerStatus().getHost();
- tokens.put(String.format(COMPONENT_IP,
+ tokens.put(String.format(COMPONENT_INSTANCE_IP,
instance.getCompInstanceName().toUpperCase()), ip);
- tokens.put(String.format(COMPONENT_HOST,
+ tokens.put(String.format(COMPONENT_INSTANCE_HOST,
instance.getCompInstanceName().toUpperCase()), host);
}
}
return tokens;
}
- private void incRunningContainers() {
+ public void incRunningContainers() {
componentMetrics.containersRunning.incr();
scheduler.getServiceMetrics().containersRunning.incr();
}
+ public void decRunningContainers() {
+ componentMetrics.containersRunning.decr();
+ scheduler.getServiceMetrics().containersRunning.decr();
+ }
+
public void incContainersReady() {
componentMetrics.containersReady.incr();
+ scheduler.getServiceMetrics().containersReady.incr();
}
public void decContainersReady() {
componentMetrics.containersReady.decr();
- }
-
- private void decRunningContainers() {
- componentMetrics.containersRunning.decr();
- scheduler.getServiceMetrics().containersRunning.decr();
+ scheduler.getServiceMetrics().containersReady.decr();
}
public int getNumReadyInstances() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 6e585aa..2492454 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -148,7 +148,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
new ContainerStatusRetriever(compInstance.scheduler,
compInstance.getContainerId(), compInstance), 0, 1,
TimeUnit.SECONDS);
-
+ compInstance.component.incRunningContainers();
long containerStartTime = System.currentTimeMillis();
try {
ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
@@ -186,6 +186,10 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
ComponentInstanceEvent event) {
compInstance.component.incContainersReady();
compInstance.containerSpec.setState(ContainerState.READY);
+ if (compInstance.timelineServiceEnabled) {
+ compInstance.serviceTimelinePublisher
+ .componentInstanceBecomeReady(compInstance.containerSpec);
+ }
}
}
@@ -225,6 +229,10 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
.getDiagnostics();
compInstance.diagnostics.append(containerDiag + System.lineSeparator());
+ if (compInstance.getState().equals(READY)) {
+ compInstance.component.decContainersReady();
+ }
+ compInstance.component.decRunningContainers();
boolean shouldExit = false;
// check if it exceeds the failure threshold
if (comp.currentContainerFailure.get() > comp.maxContainerFailurePerComp) {
@@ -250,10 +258,8 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
.submit(compInstance::cleanupRegistry);
if (compInstance.timelineServiceEnabled) {
// record in ATS
- compInstance.serviceTimelinePublisher
- .componentInstanceFinished(compInstance,
- event.getStatus().getExitStatus(), event.getStatus().getState(),
- containerDiag);
+ compInstance.serviceTimelinePublisher.componentInstanceFinished
+ (compInstance, event.getStatus().getExitStatus(), containerDiag);
}
compInstance.containerSpec.setState(ContainerState.STOPPED);
}
@@ -336,7 +342,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
container.setIp(StringUtils.join(",", status.getIPs()));
container.setHostname(status.getHost());
if (timelineServiceEnabled) {
- serviceTimelinePublisher.componentInstanceUpdated(container);
+ serviceTimelinePublisher.componentInstanceIPHostUpdated(container);
}
}
updateServiceRecord(yarnRegistryOperations, status);
@@ -410,7 +416,10 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
}
- // Release the container , cleanup registry, hdfs dir, and record in ATS
+ // Called when user flexed down the container and ContainerStoppedTransition
+ // is not executed in this case.
+ // Release the container, dec running,
+ // cleanup registry, hdfs dir, and send record to ATS
public void destroy() {
LOG.info(getCompInstanceId() + ": Flexed down by user, destroying.");
diagnostics.append(getCompInstanceId() + ": Flexed down by user");
@@ -420,10 +429,18 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
.releaseAssignedContainer(container.getId());
getCompSpec().removeContainer(containerSpec);
}
+ // update metrics
+ if (getState() == STARTED) {
+ component.decRunningContainers();
+ }
+ if (getState() == READY) {
+ component.decContainersReady();
+ component.decRunningContainers();
+ }
+
if (timelineServiceEnabled) {
- serviceTimelinePublisher
- .componentInstanceFinished(this, KILLED_BY_APPMASTER, COMPLETE,
- diagnostics.toString());
+ serviceTimelinePublisher.componentInstanceFinished(this,
+ KILLED_BY_APPMASTER, diagnostics.toString());
}
scheduler.executorService.submit(this::cleanupRegistryAndCompHdfsDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 1968e95..a7bd58d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -28,7 +28,7 @@ public class YarnServiceConf {
// Retry settings for container failures
public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
- public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval";
+ public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval-ms";
public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
@@ -40,6 +40,13 @@ public class YarnServiceConf {
public static final String DEFAULT_API_SERVER_ADDRESS = "0.0.0.0:";
public static final int DEFAULT_API_SERVER_PORT = 9191;
+ public static final String FINAL_LOG_INCLUSION_PATTERN = "yarn.service.log.include-pattern";
+ public static final String FINAL_LOG_EXCLUSION_PATTERN = "yarn.service.log.exclude-pattern";
+
+ public static final String ROLLING_LOG_INCLUSION_PATTERN = "yarn.service.rolling-log.include-pattern";
+ public static final String ROLLING_LOG_EXCLUSION_PATTERN = "yarn.service.rolling-log.exclude-pattern";
+
+
/**
* The yarn service base path:
* Defaults to HomeDir/.yarn/
@@ -98,4 +105,9 @@ public class YarnServiceConf {
Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
return userConf.getPropertyInt(name, systemConf.getInt(name, defaultValue));
}
+
+ public static String get(String name, String defaultVal,
+ Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
+ return userConf.getProperty(name, systemConf.get(name, defaultVal));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
index e5ed703..3973759 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
@@ -87,4 +87,6 @@ public interface YarnServiceConstants {
String OUT_FILE = "stdout.txt";
String ERR_FILE = "stderr.txt";
+
+ String CONTENT = "content";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
index 684f655..c4f63ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
@@ -70,9 +70,9 @@ public final class MonitorUtils {
try {
switch (readinessCheck.getType()) {
case HTTP:
- return HttpProbe.create(readinessCheck.getProps());
+ return HttpProbe.create(readinessCheck.getProperties());
case PORT:
- return PortProbe.create(readinessCheck.getProps());
+ return PortProbe.create(readinessCheck.getProperties());
default:
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
index 0d11be2..75c647b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -27,10 +27,13 @@ import org.apache.hadoop.yarn.service.utils.SliderUtils;
import java.io.IOException;
import java.nio.file.Paths;
+import java.text.MessageFormat;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConstants.CONTENT;
+
public abstract class AbstractClientProvider {
public AbstractClientProvider() {
@@ -84,11 +87,15 @@ public abstract class AbstractClientProvider {
throw new IllegalArgumentException("File type is empty");
}
- if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
- .isEmpty(file.getSrcFile())) {
- throw new IllegalArgumentException(
- "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
-
+ if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE)) {
+ if (StringUtils.isEmpty(file.getSrcFile()) &&
+ !file.getProperties().containsKey(CONTENT)) {
+ throw new IllegalArgumentException(MessageFormat.format("For {0} " +
+ "format, either src_file must be specified in ConfigFile," +
+ " or the \"{1}\" key must be specified in " +
+ "the 'properties' field of ConfigFile. ",
+ ConfigFile.TypeEnum.TEMPLATE, CONTENT));
+ }
}
if (!StringUtils.isEmpty(file.getSrcFile())) {
Path p = new Path(file.getSrcFile());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 63fbaae..582c5ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -272,8 +272,8 @@ public class ProviderUtils implements YarnServiceConstants {
break;
}
} else {
- // non-template
- resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
+ // If src_file is not specified
+ resolvePropsInConfigFileAndSaveOnHdfs(fs, tokensForSubstitution,
instance, configFile, fileName, remoteFile);
}
}
@@ -297,17 +297,17 @@ public class ProviderUtils implements YarnServiceConstants {
}
}
- private static void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
+ private static void resolvePropsInConfigFileAndSaveOnHdfs(SliderFileSystem fs,
Map<String, String> tokensForSubstitution, ComponentInstance instance,
ConfigFile configFile, String fileName, Path remoteFile)
throws IOException {
// substitute non-template configs
- substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
+ substituteMapWithTokens(configFile.getProperties(), tokensForSubstitution);
// write configs onto hdfs
PublishedConfiguration publishedConfiguration =
new PublishedConfiguration(fileName,
- configFile.getProps().entrySet());
+ configFile.getProperties().entrySet());
if (!fs.getFileSystem().exists(remoteFile)) {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(
@@ -343,7 +343,7 @@ public class ProviderUtils implements YarnServiceConstants {
confCopy.set(entry.getKey(), entry.getValue());
}
// substitute properties
- for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
+ for (Map.Entry<String, String> entry : configFile.getProperties().entrySet()) {
confCopy.set(entry.getKey(), entry.getValue());
}
// substitute env variables
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
index 7f7f9a1..6c3428a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
@@ -30,5 +30,7 @@ public enum ServiceTimelineEvent {
COMPONENT_INSTANCE_UNREGISTERED,
- COMPONENT_INSTANCE_UPDATED
+ COMPONENT_INSTANCE_IP_HOST_UPDATE,
+
+ COMPONENT_INSTANCE_BECOME_READY
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index cfe7c1b..c522986 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -20,19 +20,13 @@ package org.apache.hadoop.yarn.service.timelineservice;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.*;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.slf4j.Logger;
@@ -46,6 +40,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import static org.apache.hadoop.yarn.service.api.records.ContainerState.READY;
+import static org.apache.hadoop.yarn.service.api.records.ContainerState.STOPPED;
import static org.apache.hadoop.yarn.service.timelineservice.ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO;
/**
@@ -117,8 +113,8 @@ public class ServiceTimelinePublisher extends CompositeService {
// publish system config - YarnConfiguration
populateTimelineEntity(systemConf.iterator(), service.getId(),
ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
- // publish user conf
- publishUserConf(service.getConfiguration(), service.getId(),
+ // publish container conf
+ publishContainerConf(service.getConfiguration(), service.getId(),
ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
// publish component as separate entity.
@@ -183,7 +179,7 @@ public class ServiceTimelinePublisher extends CompositeService {
}
public void componentInstanceFinished(ComponentInstance instance,
- int exitCode, ContainerState state, String diagnostics) {
+ int exitCode, String diagnostics) {
TimelineEntity entity = createComponentInstanceEntity(
instance.getContainer().getId().toString());
@@ -192,7 +188,7 @@ public class ServiceTimelinePublisher extends CompositeService {
entityInfos.put(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE,
exitCode);
entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
- entityInfos.put(ServiceTimelineMetricsConstants.STATE, state);
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE, STOPPED);
entity.addInfo(entityInfos);
// add an event
@@ -205,7 +201,7 @@ public class ServiceTimelinePublisher extends CompositeService {
putEntity(entity);
}
- public void componentInstanceUpdated(Container container) {
+ public void componentInstanceIPHostUpdated(Container container) {
TimelineEntity entity = createComponentInstanceEntity(container.getId());
// create info keys
@@ -218,14 +214,27 @@ public class ServiceTimelinePublisher extends CompositeService {
entity.addInfo(entityInfos);
TimelineEvent updateEvent = new TimelineEvent();
- updateEvent
- .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UPDATED.toString());
+ updateEvent.setId(ServiceTimelineEvent.COMPONENT_INSTANCE_IP_HOST_UPDATE
+ .toString());
updateEvent.setTimestamp(System.currentTimeMillis());
entity.addEvent(updateEvent);
putEntity(entity);
}
+ public void componentInstanceBecomeReady(Container container) {
+ TimelineEntity entity = createComponentInstanceEntity(container.getId());
+ Map<String, Object> entityInfo = new HashMap<>();
+ entityInfo.put(ServiceTimelineMetricsConstants.STATE, READY);
+ entity.addInfo(entityInfo);
+ TimelineEvent updateEvent = new TimelineEvent();
+ updateEvent.setId(ServiceTimelineEvent.COMPONENT_INSTANCE_BECOME_READY
+ .toString());
+ updateEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(updateEvent);
+ putEntity(entity);
+ }
+
private void publishComponents(List<Component> components) {
long currentTimeMillis = System.currentTimeMillis();
for (Component component : components) {
@@ -266,22 +275,19 @@ public class ServiceTimelinePublisher extends CompositeService {
putEntity(entity);
- // publish component specific configurations
- publishUserConf(component.getConfiguration(), component.getName(),
+ // publish container specific configurations
+ publishContainerConf(component.getConfiguration(), component.getName(),
ServiceTimelineEntityType.COMPONENT.toString());
}
}
- private void publishUserConf(Configuration configuration,
+ private void publishContainerConf(Configuration configuration,
String entityId, String entityType) {
- populateTimelineEntity(configuration.getProperties().entrySet().iterator(),
- entityId, entityType);
-
populateTimelineEntity(configuration.getEnv().entrySet().iterator(),
entityId, entityType);
for (ConfigFile configFile : configuration.getFiles()) {
- populateTimelineEntity(configFile.getProps().entrySet().iterator(),
+ populateTimelineEntity(configFile.getProperties().entrySet().iterator(),
entityId, entityType);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
index 8739382..18318aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
@@ -96,9 +96,9 @@ public class TestAppJsonResolve extends Assert {
props.put("k1", "overridden");
props.put("k2", "v2");
files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
+ .PROPERTIES).properties(props));
files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
+ .XML).properties(Collections.singletonMap("k3", "v3")));
assertTrue(files.contains(simple.getFiles().get(0)));
assertTrue(files.contains(simple.getFiles().get(1)));
@@ -113,9 +113,9 @@ public class TestAppJsonResolve extends Assert {
props.put("k1", "v1");
files.clear();
files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
+ .PROPERTIES).properties(props));
files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
+ .XML).properties(Collections.singletonMap("k3", "v3")));
assertTrue(files.contains(master.getFiles().get(0)));
assertTrue(files.contains(master.getFiles().get(1)));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
index e7c7600..80b4f51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -160,7 +160,7 @@ public class TestServiceTimelinePublisher {
// updated container state
container.setState(ContainerState.READY);
- serviceTimelinePublisher.componentInstanceUpdated(container);
+ serviceTimelinePublisher.componentInstanceIPHostUpdated(container);
lastPublishedEntities =
((DummyTimelineClient) timelineClient).getLastPublishedEntities();
assertEquals(1, lastPublishedEntities.size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
index 753a9cd..30fa0eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
@@ -10,7 +10,7 @@
{
"type": "PROPERTIES",
"dest_file": "file1",
- "props": {
+ "properties": {
"k1": "v1",
"k2": "v2"
}
@@ -18,7 +18,7 @@
{
"type": "XML",
"dest_file": "file2",
- "props": {
+ "properties": {
"k3": "v3"
}
}
@@ -38,7 +38,7 @@
{
"type": "PROPERTIES",
"dest_file": "file1",
- "props": {
+ "properties": {
"k1": "overridden"
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
new file mode 100644
index 0000000..561df7f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
@@ -0,0 +1,165 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+This document describes how to configure the services to be deployed on YARN
+
+There are mainly three types of configurations:
+
+* The configurations specific to the custom service running on YARN . E.g. the hbase-site.xml for a Hbase service running on YARN.
+ * It can be specified at both global service level(`Service#configuration`) or component level(`Component#configuration`).
+ * Service-level configs are considered as the default configs for all components and component-level config can override service level config.
+ * All config properties that uses constant variables as described below are subject to substitutions.
+* The configurations specific to YARN service AM. (`Configuration#properties`).
+ * E.g. The `yarn.service.am-restart.max-attempts` which controls how many times
+the framework AM can be retried if it fails. These configs are mainly to control the behavior of the framework AM , rather than the custom services launched by the framework.
+* Some constants such as `SERVICE_NAME` for referring some system properties.
+ * They are substituted by the service AM before writing the config files.
+
+Below describes the details for each type of configurations.
+
+## Configuration for custom service
+Below is how a configuration object typically looks like:
+```
+"configuration" : {
+ "properties" : {
+ "yarn.service.am-restart.max-attempts" : 10 // config for the service AM
+ },
+ "env" : { // The environment variables to be exported when container gets launched
+ "env1" : "val1"
+ },
+ "files" : [ // The list of configuration files to be mounted for the container
+ {
+ "type": "HADOOP_XML", // The format of the config file into which the "properties" are dumped
+ "dest_file": "/etc/hadoop/conf/core-site.xml", // The location where the config file is mounted inside the container
+ "properties" : { // The list of key/value pairs to be dumped into the config file
+ "fs.defaultFS" : "hdfs://myhdfs" // This property will be written into core-site.xml
+ }
+ },
+ {
+ "type": "HADOOP_XML", // The format of the config file.
+ "src_file" : ""hdfs://mycluster/tmp/conf/yarn-site.xml"" // The location of the source config file to be downloaded
+ "dest_file": "/etc/hadoop/conf/yarn-site.xml", // The location where the config file will be mounted inside the container/
+ "properties" : {
+ "yarn.resourcemanager.webapp.address" : "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}" // Merge into (or override existing property in) yarn-site.xml
+ }
+ }
+ ]
+}
+```
+
+* properties: the configurations for service AM. Details below.
+* env : the environment variables to be exported when container gets launched.
+* files : The list of configuration files to be mounted inside the container.
+ * type: The format of the config file(`dest_file`) to be mounted inside the container. If `src_file` is specified, it is also the format for both `src_file` and `dest_file`.
+ * HADOOP_XML : the hadoop xml format. If `src_file` is specified, the file content will be read as parsed in hadoop xml format.
+ * XML : the standard xml format
+ * JSON : the standard JSON format
+ * YAML : the YAML format
+ * PROPERTIES : the java PROPERTIES format
+ * TEMPLATE : the plain text format. If `src_file` is specified, the content of the `src_file` will be written into `dest_file` post constant substitution.
+ If `src_file` is not specified, use `content` as the key in `properties` field, and the value will be the actual content to be written in the `dest_file` post constant substitution. E.g
+
+ ```
+ {
+ "type": "TEMPLATE"
+ "dest_file": "/etc/conf/hello"
+ "properties" : {
+ "content" : "Hello world"
+ }
+ }
+ ```
+ The string `Hello world` will be written into a file located at `/etc/conf/hello` inside the container.
+ * src_file : [optional], the source location of the config file at a network accessible location such as hdfs.
+ * The format of both `src_file` and `dest_file` are defined by `type`.
+ * Currently, It only works with `HADOOP_XML` and `TEMPLATE` type.
+ * The `src_file` will be downloaded by YARN NodeManager and be mounted inside the container as in the location specified by `dest_file`.
+ * If any properties specified in the `properties` field, they are added into (or overwriting existing properties in) the `src_file`.
+ * If `src_file` is not specified, only the properties in the `properties` field will be written into the `dest_file` file.
+ * dest_file : the location where the config file is mounted inside the container. The file format is defined by `type`.
+ dest_file can be an absolute path or a relative path. If it's a relative path, the file will be located in the `$PWD/conf` directory (where `$PWD` is the container local directory which is mounted to all docker containers launched by YARN)
+ * properties : The list of key/value pair configs to be written into the `dest_file` in the format as defined in `type`. If `src_file` is specified, these properties will be added into (or overwriting existing properties in) the `src_file`.
+
+## Configuration for YARN service AM
+This section describes the configurations for configuring the YARN service AM.
+
+These can be specified either in the cluster `yarn-site.xml` at the global level or in the `properties` field of the `Configuration` object as per service basis like below:
+```
+{
+ "configuration" : {
+ "properties" : {
+ "yarn.service.am-restart.max-attempts" : 10
+ }
+ }
+}
+```
+Above config make the service AM to be retried at max 10 times.
+
+#### Available configurations:
+
+| Name | Description |
+| ------------ | ------------- |
+|yarn.service.client-am.retry.max-wait-ms | the max retry time in milliseconds for the service client to talk to the service AM. By default, it is set to 0, which means no retry |
+|yarn.service.client-am.retry-interval-ms | the retry interval in milliseconds for the service client to talk to the service AM. By default, it is 2000, i.e. 2 seconds |
+|yarn.service.container-failure.retry.max | the max number of retries for the container to be auto restarted if it fails. By default, it is set to -1, which means forever.
+|yarn.service.container-failure.retry-interval-ms| the retry interval in milliseconds for the container to be restarted. By default, it is 30000, i.e. 30 seconds |
+|yarn.service.am-restart.max-attempts| the max number of attempts for the framework AM
+|yarn.service.am-resource.memory | the memory size in GB for the framework AM. By default, it is set to 1024
+|yarn.service.queue | the default queue to which the service will be submitted. By default, it is submitted to `default` queue
+|yarn.service.base.path | the root location for the service artifacts on hdfs for a user. By default, it is under ${user_home_dir}/.yarn/
+|yarn.service.container-failure-per-component.threshold | the max number of container failures for a given component before the AM exits.
+|yarn.service.node-blacklist.threshold | Maximum number of container failures on a node before the node is blacklisted by the AM
+|yarn.service.failure-count-reset.window | The interval in seconds when the `yarn.service.container-failure-per-component.threshold` and `yarn.service.node-blacklist.threshold` gets reset. By default, it is 21600, i.e. 6 hours
+|yarn.service.readiness-check-interval.seconds | The interval in seconds between readiness checks. By default, it is 30 seconds
+|yarn.service.log.include-pattern| The regex expression for including log files whose file name matches it when aggregating the logs after the application completes.
+|yarn.service.log.exclude-pattern| The regex expression for excluding log files whose file name matches it when aggregating the logs after the application completes. If the log file name matches both include and exclude pattern, this file will be excluded.
+|yarn.service.rolling-log.include-pattern| The regex expression for including log files whose file name matches it when aggregating the logs while app is running.
+|yarn.service.rolling-log.exclude-pattern| The regex expression for excluding log files whose file name matches it when aggregating the logs while app is running. If the log file name matches both include and exclude pattern, this file will be excluded.
+
+## Constant variables for custom service
+The service framework provides some constant variables for user to configure their services. These variables are either dynamically generated by the system or are static ones such as service name defined by the user.
+User can use these constants in their configurations to be dynamically substituted by the service AM.E.g.
+```
+{
+ "type" : "HADOOP_XML",
+ "dest_file" : "/etc/hadoop/hbase-site.xml",
+ "properties" : {
+ "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+}
+```
+Here, `COMPONENT_INSTANCE_NAME` and `SERVICE_NAME` are the constants to be substituted by the system.
+
+Suppose the `COMPONENT_INSTANCE_NAME` is `regionserver-0` and `SERVICE_NAME` is defined by user as `hbase`, user name is `devuser` and domain name is `hwxdev.site`.
+Then, the config will be substituted by the service AM and written in the config file `/etc/hadoop/hbase-site.xml` inside the container as below:
+```
+<property>
+ <name>hbase.regionserver.hostname</name>
+ <value>regionserver-0.hbase.devuser.hwxdev.site</value>
+</property>
+```
+where `regionserver-0` is the actual component instance name assigned by the system for this container.
+
+#### Available constants:
+| Name | Description |
+| ------------ | ------------- |
+| SERVICE_NAME | name of the service defined by the user
+| USER | user who submits the service |
+| DOMAIN | the domain name for the cluster |
+| COMPONENT_NAME | the name for a given component |
+| COMPONENT_INSTANCE_NAME | the name for a given component instance (i.e. container) |
+| COMPONENT_ID | the monotonically increasing integer for a given component
+| CONTAINER_ID | the YARN container Id for a given container |
+| ${COMPONENT_INSTANCE_NAME}_HOST | the hostname for a component instance (i.e. container), e.g. REGIONSERVER-0_HOST will be substituted by the actual hostname of the component instance. Note all characters must be uppercase. |
+| ${COMPONENT_INSTANCE_NAME}_IP | the ip for a component instance (i.e. container), e.g. REGIONSERVER-0_IP will be substituted by the actual IP address of the component instance. Note all characters must be uppercase. |
+| CLUSTER_FS_URI | the URI of the cluster hdfs |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
index e4881ce..0ddd126 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -45,7 +45,7 @@ Note this example requires registry DNS.
{
"type": "ENV",
"dest_file": "/var/www/html/index.html",
- "props": {
+ "properties": {
"content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
}
}
@@ -122,7 +122,7 @@ A readiness check is added for the `httpd` component:
```
"readiness_check": {
"type": "HTTP",
- "props": {
+ "properties": {
"url": "http://${THIS_HOST}:8080"
}
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
index 58daee5..72c2d3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -52,9 +52,10 @@ The benefits of combining these workloads are two-fold:
* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN.
* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN.
-* [Service Discovery](ServiceDiscovery.md): Descirbes the service discovery mechanism on YARN.
+* [Service Discovery](ServiceDiscovery.md): Describes the service discovery mechanism on YARN.
* [Registry DNS](RegistryDNS.md): Deep dives into the Registry DNS internals.
* [Examples](Examples.md): List some example service definitions (`Yarnfile`).
+* [Configurations](Configurations.md): Describes how to configure the custom services on YARN.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index 7a2490e..c0e12c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -251,7 +251,7 @@ A config file that needs to be created and made available as a volume in a servi
|type|Config file in the standard format like xml, properties, json, yaml, template.|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, ENV, HADOOP_XML)||
|dest_file|The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.|false|string||
|src_file|This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.|false|string||
-|props|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
+|properties|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
### Configuration
@@ -260,7 +260,7 @@ Set of configuration properties that can be injected into the service components
|Name|Description|Required|Schema|Default|
|----|----|----|----|----|
-|properties|A blob of key-value pairs of common service properties.|false|object||
+|properties|A blob of key-value pairs for configuring YARN service AM.|false|object||
|env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object||
|files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array||
@@ -308,7 +308,7 @@ A custom command or a pluggable helper container to determine the readiness of a
|Name|Description|Required|Schema|Default|
|----|----|----|----|----|
|type|E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).|true|enum (HTTP, PORT)||
-|props|A blob of key value pairs that will be used to configure the check.|false|object||
+|properties|A blob of key value pairs that will be used to configure the check.|false|object||
|artifact|Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET|false|Artifact||
@@ -529,14 +529,14 @@ POST URL - http://localhost:8088:/ws/v1/services/hbase-app-1
{
"type": "XML",
"dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
+ "properties": {
"fs.defaultFS": "${CLUSTER_FS_URI}"
}
},
{
"type": "XML",
"dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
+ "properties": {
"hbase.cluster.distributed": "true",
"hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
"hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
@@ -569,14 +569,14 @@ POST URL - http://localhost:8088:/ws/v1/services/hbase-app-1
{
"type": "XML",
"dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
+ "properties": {
"fs.defaultFS": "${CLUSTER_FS_URI}"
}
},
{
"type": "XML",
"dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
+ "properties": {
"hbase.cluster.distributed": "true",
"hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
"hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
index b4bac0d..812f1e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
@@ -49,7 +49,7 @@ export default Ember.Controller.extend({
contentPath: 'memory'
}, {
id: 'instances',
- headerTitle: 'Number Of Instances',
+ headerTitle: 'Number of Instances',
contentPath: 'instances',
observePath: true
});
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
new file mode 100644
index 0000000..6e6f4dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -0,0 +1,1654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+ /**
+ * Atomic bool to track whether or not process security has already been
+ * turned on (prevents re-entrancy)
+ */
+ private static final AtomicBoolean processSecurityAlreadyInitialized =
+ new AtomicBoolean(false);
+ public static final String JAVA_SECURITY_KRB5_REALM =
+ "java.security.krb5.realm";
+ public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
+
+ /**
+ * Winutils
+ */
+ public static final String WINUTILS = "WINUTILS.EXE";
+ /**
+ * name of openssl program
+ */
+ public static final String OPENSSL = "openssl";
+
+ /**
+ * name of python program
+ */
+ public static final String PYTHON = "python";
+
+ /**
+ * type of docker standalone service
+ */
+ public static final String DOCKER = "docker";
+ /**
+ * type of docker on yarn service
+ */
+ public static final String DOCKER_YARN = "yarn_docker";
+
+ public static final int NODE_LIST_LIMIT = 10;
+
+ private SliderUtils() {
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s string
+ * @return true iff the string is neither null nor empty
+ */
+ public static boolean isUnset(String s) {
+ return s == null || s.isEmpty();
+ }
+
+ public static boolean isSet(String s) {
+ return !isUnset(s);
+ }
+
+ public static boolean isEmpty(Collection l) {
+ return l == null || l.isEmpty();
+ }
+
+ /**
+ * Probe for a collection existing and not being empty
+ * @param l collection
+ * @return true if the reference is valid and it contains entries
+ */
+
+ public static boolean isNotEmpty(Collection l) {
+ return l != null && !l.isEmpty();
+ }
+
+ /**
+ * Probe for a map existing and not being empty
+ * @param m map
+ * @return true if the reference is valid and it contains map entries
+ */
+ public static boolean isNotEmpty(Map m) {
+ return m != null && !m.isEmpty();
+ }
+
+ /*
+ * Validates whether num is an integer
+ * @param num
+ * @param msg the message to be shown in exception
+ */
+ @SuppressWarnings("ResultOfMethodCallIgnored")
+ private static void validateNumber(String num, String msg) throws
+ BadConfigException {
+ try {
+ Integer.parseInt(num);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(msg + num);
+ }
+ }
+
+ /*
+ * Translates the trailing JVM heapsize unit: g, G, m, M
+ * This assumes designated unit of 'm'
+ * @param heapsize
+ * @return heapsize in MB
+ */
+ public static String translateTrailingHeapUnit(String heapsize) throws
+ BadConfigException {
+ String errMsg = "Bad heapsize: ";
+ if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
+ String num = heapsize.substring(0, heapsize.length() - 1);
+ validateNumber(num, errMsg);
+ return num;
+ }
+ if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
+ String num = heapsize.substring(0, heapsize.length() - 1) + "000";
+ validateNumber(num, errMsg);
+ return num;
+ }
+ // check if specified heap size is a number
+ validateNumber(heapsize, errMsg);
+ return heapsize;
+ }
+
+ /**
+ * recursive directory delete
+ * @param dir dir to delete
+ * @throws IOException on any problem
+ */
+ public static void deleteDirectoryTree(File dir) throws IOException {
+ if (dir.exists()) {
+ if (dir.isDirectory()) {
+ log.info("Cleaning up {}", dir);
+ //delete the children
+ File[] files = dir.listFiles();
+ if (files == null) {
+ throw new IOException("listfiles() failed for " + dir);
+ }
+ for (File file : files) {
+ log.info("deleting {}", file);
+ if (!file.delete()) {
+ log.warn("Unable to delete " + file);
+ }
+ }
+ if (!dir.delete()) {
+ log.warn("Unable to delete " + dir);
+ }
+ } else {
+ throw new IOException("Not a directory " + dir);
+ }
+ } else {
+ //not found, do nothing
+ log.debug("No output dir yet");
+ }
+ }
+
+ /**
+ * Find a containing JAR
+ * @param clazz class to find
+ * @return the file
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ * @throws FileNotFoundException if the class did not resolve to a file
+ */
+ public static File findContainingJarOrFail(Class clazz) throws IOException {
+ File localFile = SliderUtils.findContainingJar(clazz);
+ if (null == localFile) {
+ throw new FileNotFoundException("Could not find JAR containing " + clazz);
+ }
+ return localFile;
+ }
+
+
+ /**
+ * Find a containing JAR
+ * @param my_class class to find
+ * @return the file or null if it is not found
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ */
+ public static File findContainingJar(Class my_class) throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ if (loader == null) {
+ throw new IOException(
+ "Class " + my_class + " does not have a classloader!");
+ }
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+ Enumeration<URL> urlEnumeration = loader.getResources(class_file);
+ for (; urlEnumeration.hasMoreElements(); ) {
+ URL url = urlEnumeration.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ String jarFilePath = toReturn.replaceAll("!.*$", "");
+ return new File(jarFilePath);
+ } else {
+ log.info("could not locate JAR containing {} URL={}", my_class, url);
+ }
+ }
+ return null;
+ }
+
+ public static void checkPort(String hostname, int port, int connectTimeout)
+ throws IOException {
+ InetSocketAddress addr = new InetSocketAddress(hostname, port);
+ checkPort(hostname, addr, connectTimeout);
+ }
+
+ @SuppressWarnings("SocketOpenedButNotSafelyClosed")
+ public static void checkPort(String name,
+ InetSocketAddress address,
+ int connectTimeout)
+ throws IOException {
+ try(Socket socket = new Socket()) {
+ socket.connect(address, connectTimeout);
+ } catch (Exception e) {
+ throw new IOException("Failed to connect to " + name
+ + " at " + address
+ + " after " + connectTimeout + "milliseconds"
+ + ": " + e,
+ e);
+ }
+ }
+
+ public static void checkURL(String name, String url, int timeout) throws
+ IOException {
+ InetSocketAddress address = NetUtils.createSocketAddr(url);
+ checkPort(name, address, timeout);
+ }
+
+ /**
+ * A required file
+ * @param role role of the file (for errors)
+ * @param filename the filename
+ * @throws ExitUtil.ExitException if the file is missing
+ * @return the file
+ */
+ public static File requiredFile(String filename, String role) throws
+ IOException {
+ if (filename.isEmpty()) {
+ throw new ExitUtil.ExitException(-1, role + " file not defined");
+ }
+ File file = new File(filename);
+ if (!file.exists()) {
+ throw new ExitUtil.ExitException(-1,
+ role + " file not found: " +
+ file.getCanonicalPath());
+ }
+ return file;
+ }
+
+ private static final PatternValidator clusternamePattern
+ = new PatternValidator("[a-z][a-z0-9_-]*");
+
+ /**
+ * Normalize a cluster name then verify that it is valid
+ * @param name proposed cluster name
+ * @return true iff it is valid
+ */
+ public static boolean isClusternameValid(String name) {
+ return name != null && clusternamePattern.matches(name);
+ }
+
+ /**
+ * Copy a directory to a new FS -both paths must be qualified. If
+ * a directory needs to be created, supplied permissions can override
+ * the default values. Existing directories are not touched
+ * @param conf conf file
+ * @param srcDirPath src dir
+ * @param destDirPath dest dir
+ * @param permission permission for the dest directory; null means "default"
+ * @return # of files copies
+ */
+ @SuppressWarnings("deprecation")
+ public static int copyDirectory(Configuration conf,
+ Path srcDirPath,
+ Path destDirPath,
+ FsPermission permission) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
+ FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcDirPath)) {
+ throw new FileNotFoundException("Source dir not found " + srcDirPath);
+ }
+ if (!srcFS.isDirectory(srcDirPath)) {
+ throw new FileNotFoundException(
+ "Source dir not a directory " + srcDirPath);
+ }
+ GlobFilter dotFilter = new GlobFilter("[!.]*");
+ FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
+ int srcFileCount = entries.length;
+ if (srcFileCount == 0) {
+ return 0;
+ }
+ if (permission == null) {
+ permission = FsPermission.getDirDefault();
+ }
+ if (!destFS.exists(destDirPath)) {
+ new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
+ permission);
+ }
+ Path[] sourcePaths = new Path[srcFileCount];
+ for (int i = 0; i < srcFileCount; i++) {
+ FileStatus e = entries[i];
+ Path srcFile = e.getPath();
+ if (srcFS.isDirectory(srcFile)) {
+ String msg = "Configuration dir " + srcDirPath
+ + " contains a directory " + srcFile;
+ log.warn(msg);
+ throw new IOException(msg);
+ }
+ log.debug("copying src conf file {}", srcFile);
+ sourcePaths[i] = srcFile;
+ }
+ log.debug("Copying {} files from {} to dest {}", srcFileCount,
+ srcDirPath,
+ destDirPath);
+ FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
+ return srcFileCount;
+ }
+
+ /**
+ * Copy a file to a new FS -both paths must be qualified.
+ * @param conf conf file
+ * @param srcFile src file
+ * @param destFile dest file
+ */
+ @SuppressWarnings("deprecation")
+ public static void copy(Configuration conf,
+ Path srcFile,
+ Path destFile) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcFile)) {
+ throw new FileNotFoundException("Source file not found " + srcFile);
+ }
+ if (!srcFS.isFile(srcFile)) {
+ throw new FileNotFoundException(
+ "Source file not a file " + srcFile);
+ }
+ FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
+ FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
+ }
+
+ /**
+ * Take a collection, return a list containing the string value of every
+ * element in the collection.
+ * @param c collection
+ * @return a stringified list
+ */
+ public static List<String> collectionToStringList(Collection c) {
+ List<String> l = new ArrayList<>(c.size());
+ for (Object o : c) {
+ l.add(o.toString());
+ }
+ return l;
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(Collection collection, String separator) {
+ return join(collection, separator, true);
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
+ StringBuilder b = new StringBuilder();
+ // fast return on empty collection
+ if (collection.isEmpty()) {
+ return trailing ? separator : "";
+ }
+ for (Object o : collection) {
+ b.append(o);
+ b.append(separator);
+ }
+ int length = separator.length();
+ String s = b.toString();
+ return (trailing || s.isEmpty()) ?
+ s : (b.substring(0, b.length() - length));
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator) {
+ return join(collection, separator, true);
+
+
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection strings
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator,
+ boolean trailing) {
+ return join(Arrays.asList(collection), separator, trailing);
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -except at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the list
+ */
+ public static String joinWithInnerSeparator(String separator,
+ Object... collection) {
+ StringBuilder b = new StringBuilder();
+ boolean first = true;
+
+ for (Object o : collection) {
+ if (first) {
+ first = false;
+ } else {
+ b.append(separator);
+ }
+ b.append(o.toString());
+ b.append(separator);
+ }
+ return b.toString();
+ }
+
+ /**
+ * Resolve a mandatory environment variable
+ * @param key env var
+ * @return the resolved value
+ * @throws BadClusterStateException
+ */
+ public static String mandatoryEnvVariable(String key) throws
+ BadClusterStateException {
+ String v = System.getenv(key);
+ if (v == null) {
+ throw new BadClusterStateException("Missing Environment variable " + key);
+ }
+ return v;
+ }
+
+ public static String appReportToString(ApplicationReport r,
+ String separator) {
+ StringBuilder builder = new StringBuilder(512);
+ builder.append("service ")
+ .append(
+ r.getName())
+ .append("/")
+ .append(r.getApplicationType())
+ .append(separator);
+ Set<String> tags = r.getApplicationTags();
+ if (!tags.isEmpty()) {
+ for (String tag : tags) {
+ builder.append(tag).append(separator);
+ }
+ }
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
+ dateFormat.setTimeZone(TimeZone.getDefault());
+ builder.append("state: ").append(r.getYarnApplicationState());
+ String trackingUrl = r.getTrackingUrl();
+ if (isSet(trackingUrl)) {
+ builder.append(separator).append("URL: ").append(trackingUrl);
+ }
+ builder.append(separator)
+ .append("Started: ")
+ .append(dateFormat.format(new Date(r.getStartTime())));
+ long finishTime = r.getFinishTime();
+ if (finishTime > 0) {
+ builder.append(separator)
+ .append("Finished: ")
+ .append(dateFormat.format(new Date(finishTime)));
+ }
+ String rpcHost = r.getHost();
+ if (!isSet(rpcHost)) {
+ builder.append(separator)
+ .append("RPC :")
+ .append(rpcHost)
+ .append(':')
+ .append(r.getRpcPort());
+ }
+ String diagnostics = r.getDiagnostics();
+ if (!isSet(diagnostics)) {
+ builder.append(separator).append("Diagnostics :").append(diagnostics);
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Filter a string value given a single filter
+ *
+ * @param value
+ * the string value to check
+ * @param filter
+ * a single string filter
+ * @return return true if value should be trapped, false if it should be let
+ * through
+ */
+ public static boolean filter(String value, String filter) {
+ return !(StringUtils.isEmpty(filter) || filter.equals(value));
+ }
+
+ /**
+ * Filter a string value given a set of filters
+ *
+ * @param value
+ * the string value to check
+ * @param filters
+ * a set of string filters
+ * @return return true if value should be trapped, false if it should be let
+ * through
+ */
+ public static boolean filter(String value, Set<String> filters) {
+ return !(filters.isEmpty() || filters.contains(value));
+ }
+
+ /**
+ * Sorts the given list of service reports, most recently started
+ * or finished instance first.
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
+ Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
+ }
+
+ /**
+ * Sorts the given list of service reports
+ * Finished instances are ordered by finished time and running/accepted instances are
+ * ordered by start time
+ * Finally Instance are order by finished instances coming after running instances
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationReport(List<ApplicationReport> instances) {
+ if (instances.size() <= 1) {
+ return;
+ }
+ List<ApplicationReport> nonLiveInstance =
+ new ArrayList<>(instances.size());
+ List<ApplicationReport> liveInstance =
+ new ArrayList<>(instances.size());
+
+ for (ApplicationReport report : instances) {
+ if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
+ ||
+ report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
+ liveInstance.add(report);
+ } else {
+ nonLiveInstance.add(report);
+ }
+ }
+
+ if (liveInstance.size() > 1) {
+ Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
+ }
+ if (nonLiveInstance.size() > 1) {
+ Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
+ }
+ instances.clear();
+ instances.addAll(liveInstance);
+ instances.addAll(nonLiveInstance);
+ }
+
+ /**
+ * Merge in one map to another -all entries in the second map are
+ * merged into the first -overwriting any duplicate keys.
+ * @param first first map -the updated one.
+ * @param second the map that is merged in
+ * @return the first map
+ */
+ public static Map<String, String> mergeMap(Map<String, String> first,
+ Map<String, String> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Merge a set of entries into a map. This will take the entryset of
+ * a map, or a Hadoop collection itself
+ * @param dest destination
+ * @param entries entries
+ * @return dest -with the entries merged in
+ */
+ public static Map<String, String> mergeEntries(Map<String, String> dest,
+ Iterable<Map.Entry<String, String>> entries) {
+ for (Map.Entry<String, String> entry : entries) {
+ dest.put(entry.getKey(), entry.getValue());
+ }
+ return dest;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ Preconditions.checkArgument(first != null, "Null 'first' value");
+ Preconditions.checkArgument(second != null, "Null 'second' value");
+ for (Map.Entry<T1, T2> entry : second.entrySet()) {
+ T1 key = entry.getKey();
+ if (!first.containsKey(key)) {
+ first.put(key, entry.getValue());
+ }
+ }
+ return first;
+ }
+
+ /**
+ * Convert a map to a multi-line string for printing
+ * @param map map to stringify
+ * @return a string representation of the map
+ */
+ public static String stringifyMap(Map<String, String> map) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ builder.append(entry.getKey())
+ .append("=\"")
+ .append(entry.getValue())
+ .append("\"\n");
+
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Parse an int value, replacing it with defval if undefined;
+ * @param errorKey key to use in exceptions
+ * @param defVal default value to use if the key is not in the map
+ * @param min min value or -1 for do not check
+ * @param max max value or -1 for do not check
+ * @return the int value the integer value
+ * @throws BadConfigException if the value could not be parsed
+ */
+ public static int parseAndValidate(String errorKey,
+ String valS,
+ int defVal,
+ int min, int max) throws
+ BadConfigException {
+ if (valS == null) {
+ valS = Integer.toString(defVal);
+ }
+ String trim = valS.trim();
+ int val;
+ try {
+ val = Integer.decode(trim);
+ } catch (NumberFormatException e) {
+ throw new BadConfigException("Failed to parse value of "
+ + errorKey + ": \"" + trim + "\"");
+ }
+ if (min >= 0 && val < min) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is less than the minimum of " + min);
+ }
+ if (max >= 0 && val > max) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is more than the maximum of " + max);
+ }
+ return val;
+ }
+
+ public static InetSocketAddress getRmAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ }
+
+ public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ }
+
+ /**
+ * probe to see if the RM scheduler is defined
+ * @param conf config
+ * @return true if the RM scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isRmSchedulerAddressDefined(Configuration conf) {
+ InetSocketAddress address = getRmSchedulerAddress(conf);
+ return isAddressDefined(address);
+ }
+
+ /**
+ * probe to see if the address
+ * @param address network address
+ * @return true if the scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isAddressDefined(InetSocketAddress address) {
+ if (address == null || address.getHostString() == null) {
+ return false;
+ }
+ return !(address.getHostString().equals("0.0.0.0"));
+ }
+
+ public static void setRmAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
+ }
+
+ public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
+ }
+
+ public static boolean hasAppFinished(ApplicationReport report) {
+ return report == null ||
+ report.getYarnApplicationState().ordinal() >=
+ YarnApplicationState.FINISHED.ordinal();
+ }
+
+ public static String containerToString(Container container) {
+ if (container == null) {
+ return "null container";
+ }
+ return String.format(Locale.ENGLISH,
+ "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
+ container.getId(),
+ container.getNodeId(),
+ container.getNodeHttpAddress(),
+ container.getPriority(),
+ container.getResource());
+ }
+
+ /**
+ * convert an AM report to a string for diagnostics
+ * @param report the report
+ * @return the string value
+ */
+ public static String reportToString(ApplicationReport report) {
+ if (report == null) {
+ return "Null service report";
+ }
+
+ return "App " + report.getName() + "/" + report.getApplicationType() +
+ "# " +
+ report.getApplicationId() + " user " + report.getUser() +
+ " is in state " + report.getYarnApplicationState() +
+ " RPC: " + report.getHost() + ":" + report.getRpcPort() +
+ " URL: " + report.getOriginalTrackingUrl();
+ }
+
+ /**
+ * Convert a YARN URL into a string value of a normal URL
+ * @param url URL
+ * @return string representatin
+ */
+ public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(url.getScheme()).append("://");
+ if (url.getHost() != null) {
+ builder.append(url.getHost()).append(":").append(url.getPort());
+ }
+ builder.append(url.getFile());
+ return builder.toString();
+ }
+
+ /**
+ * Get a random open port
+ * @return true if the port was available for listening on
+ */
+ public static int getOpenPort() throws IOException {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ return socket.getLocalPort();
+ } finally {
+ if (socket != null) {
+ socket.close();
+ }
+ }
+ }
+
+ /**
+ * See if a port is available for listening on by trying to listen
+ * on it and seeing if that works or fails.
+ * @param port port to listen to
+ * @return true if the port was available for listening on
+ */
+ public static boolean isPortAvailable(int port) {
+ try {
+ ServerSocket socket = new ServerSocket(port);
+ socket.close();
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ // Build env map: key -> value;
+ // value will be replaced by the corresponding value in tokenMap, if any.
+ public static Map<String, String> buildEnvMap(
+ org.apache.hadoop.yarn.service.api.records.Configuration conf,
+ Map<String,String> tokenMap) {
+ if (tokenMap == null) {
+ return conf.getEnv();
+ }
+ Map<String, String> env = new HashMap<>();
+ for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()),
+ token.getValue());
+ }
+ env.put(key,val);
+ }
+ return env;
+ }
+
+ /**
+ * Apply a set of command line options to a cluster role map
+ * @param clusterRoleMap cluster role map to merge onto
+ * @param commandOptions command opts
+ */
+ public static void applyCommandLineRoleOptsToRoleMap(
+ Map<String, Map<String, String>> clusterRoleMap,
+ Map<String, Map<String, String>> commandOptions) {
+ for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
+ String key = entry.getKey();
+ Map<String, String> optionMap = entry.getValue();
+ Map<String, String> existingMap = clusterRoleMap.get(key);
+ if (existingMap == null) {
+ existingMap = new HashMap<String, String>();
+ }
+ log.debug("Overwriting role options with command line values {}",
+ stringifyMap(optionMap));
+ mergeMap(existingMap, optionMap);
+ //set or overwrite the role
+ clusterRoleMap.put(key, existingMap);
+ }
+ }
+
+ /**
+ * Verify that a Kerberos principal has been set -if not fail
+ * with an error message that actually tells you what is missing
+ * @param conf configuration to look at
+ * @param principal key of principal
+ * @throws BadConfigException if the key is not set
+ */
+ public static void verifyPrincipalSet(Configuration conf,
+ String principal) throws
+ BadConfigException {
+ String principalName = conf.get(principal);
+ if (principalName == null) {
+ throw new BadConfigException("Unset Kerberos principal : %s",
+ principal);
+ }
+ log.debug("Kerberos princial {}={}", principal, principalName);
+ }
+
+ /**
+ * Flag to indicate whether the cluster is in secure mode
+ * @param conf configuration to look at
+ * @return true if the slider client/service should be in secure mode
+ */
+ public static boolean isHadoopClusterSecure(Configuration conf) {
+ return SecurityUtil.getAuthenticationMethod(conf) !=
+ UserGroupInformation.AuthenticationMethod.SIMPLE;
+ }
+
+ /**
+ * Init security if the cluster configuration declares the cluster is secure
+ * @param conf configuration to look at
+ * @return true if the cluster is secure
+ * @throws IOException cluster is secure
+ * @throws SliderException the configuration/process is invalid
+ */
+ public static boolean maybeInitSecurity(Configuration conf) throws
+ IOException,
+ SliderException {
+ boolean clusterSecure = isHadoopClusterSecure(conf);
+ if (clusterSecure) {
+ log.debug("Enabling security");
+ initProcessSecurity(conf);
+ }
+ return clusterSecure;
+ }
+
+ /**
+ * Turn on security. This is setup to only run once.
+ * @param conf configuration to build up security
+ * @return true if security was initialized in this call
+ * @throws IOException IO/Net problems
+ * @throws BadConfigException the configuration and system state are inconsistent
+ */
+ public static boolean initProcessSecurity(Configuration conf) throws
+ IOException,
+ SliderException {
+
+ if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
+ //security is already inited
+ return false;
+ }
+
+ log.info("JVM initialized into secure mode with kerberos realm {}",
+ SliderUtils.getKerberosRealm());
+ //this gets UGI to reset its previous world view (i.e simple auth)
+ //security
+ log.debug("java.security.krb5.realm={}",
+ System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
+ log.debug("java.security.krb5.kdc={}",
+ System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
+ log.debug("hadoop.security.authentication={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
+ log.debug("hadoop.security.authorization={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
+ log.debug("Authenticating as {}", authUser);
+ log.debug("Login user is {}", UserGroupInformation.getLoginUser());
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED,
+ "Although secure mode is enabled," +
+ "the service has already set up its user as an insecure entity %s",
+ authUser);
+ }
+ if (authUser.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.SIMPLE) {
+ throw new BadConfigException("Auth User is not Kerberized %s" +
+ " -security has already been set up with the wrong authentication method. "
+ + "This can occur if a file system has already been created prior to the loading of "
+ + "the security configuration.",
+ authUser);
+
+ }
+
+ SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
+ SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal");
+ return true;
+ }
+
+ /**
+ * Force an early login: This catches any auth problems early rather than
+ * in RPC operations
+ * @throws IOException if the login fails
+ */
+ public static void forceLogin() throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isLoginKeytabBased()) {
+ UserGroupInformation.getLoginUser().reloginFromKeytab();
+ } else {
+ UserGroupInformation.getLoginUser().reloginFromTicketCache();
+ }
+ }
+ }
+
+ public static String getLibDir() {
+ String[] libDirs = getLibDirs();
+ if (libDirs == null || libDirs.length == 0) {
+ return null;
+ }
+ return libDirs[0];
+ }
+
+ public static String[] getLibDirs() {
+ String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
+ if (isUnset(libDirStr)) {
+ return ArrayUtils.EMPTY_STRING_ARRAY;
+ }
+ return StringUtils.split(libDirStr, ',');
+ }
+
+ /**
+ * Submit a JAR containing a specific class and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param clazz class to look for
+ * @param libdir lib directory
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public static LocalResource putJar(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
+ LocalResource res = sliderFileSystem.submitJarWithClass(
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
+ return res;
+ }
+
+ /**
+ * Submit a JAR containing and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param libDir lib directory
+ * @param srcPath copy jars from
+ */
+ public static void putAllJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String srcPath) throws IOException, SliderException {
+ log.info("Loading all dependencies from {}", srcPath);
+ if (SliderUtils.isSet(srcPath)) {
+ File srcFolder = new File(srcPath);
+ FilenameFilter jarFilter = createJarFilter();
+ File[] listOfJars = srcFolder.listFiles(jarFilter);
+ if (listOfJars == null || listOfJars.length == 0) {
+ return;
+ }
+ for (File jarFile : listOfJars) {
+ LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+ providerResources.put(libDir + "/" + jarFile.getName(), res);
+ }
+ }
+ }
+
+ /**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
+ * Submit the AM tar.gz containing all dependencies and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ */
+ public static void putAmTarGzipAndUpdate(
+ Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem
+ ) throws IOException, SliderException {
+ log.info("Loading all dependencies from {}{}",
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ sliderFileSystem.submitTarGzipAndUpdate(providerResources);
+ }
+
+ public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
+ Map<String, Map<String, String>> dest = new HashMap<>();
+ for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
+ dest.put(entry.getKey(), stringMapClone(entry.getValue()));
+ }
+ return dest;
+ }
+
+ public static Map<String, String> stringMapClone(Map<String, String> src) {
+ Map<String, String> dest = new HashMap<>();
+ return mergeEntries(dest, src.entrySet());
+ }
+
+ /**
+ * List a directory in the local filesystem
+ * @param dir directory
+ * @return a listing, one to a line
+ */
+ public static String listDir(File dir) {
+ if (dir == null) {
+ return "";
+ }
+ String[] confDirEntries = dir.list();
+ if (confDirEntries == null) {
+ return "";
+ }
+ StringBuilder builder = new StringBuilder();
+ for (String entry : confDirEntries) {
+ builder.append(entry).append("\n");
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Create a file:// path from a local file
+ * @param file file to point the path
+ * @return a new Path
+ */
+ public static Path createLocalPath(File file) {
+ return new Path(file.toURI());
+ }
+
+ public static String getKerberosRealm() {
+ try {
+ return KerberosUtil.getDefaultRealm();
+ } catch (Exception e) {
+ log.debug("introspection into JVM internals failed", e);
+ return "(unknown)";
+
+ }
+ }
+
+ /**
+ * Build up the classpath for execution
+ * -behaves very differently on a mini test cluster vs a production
+ * production one.
+ *
+ * @param sliderConfDir relative path to the dir containing slider config
+ * options to put on the classpath -or null
+ * @param libdir directory containing the JAR files
+ * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
+ * (and hence the current classpath should be used, not anything built up)
+ * @return a classpath
+ */
+ public static ClasspathConstructor buildClasspath(String sliderConfDir,
+ String libdir,
+ SliderFileSystem sliderFileSystem,
+ boolean usingMiniMRCluster) {
+
+ ClasspathConstructor classpath = new ClasspathConstructor();
+ classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+
+ // add the runtime classpath needed for tests to work
+ if (usingMiniMRCluster) {
+ // for mini cluster we pass down the java CP properties
+ // and nothing else
+ classpath.appendAll(classpath.localJVMClasspath());
+ } else {
+ if (sliderConfDir != null) {
+ classpath.addClassDirectory(sliderConfDir);
+ }
+ classpath.addLibDir(libdir);
+ if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
+ classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
+ } else {
+ log.info(
+ "For faster submission of apps, upload dependencies using cmd {} {}",
+ SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
+ }
+ classpath.addRemoteClasspathEnvVar();
+ classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
+ }
+ return classpath;
+ }
+
+ /**
+ * Verify that a path refers to a directory. If not
+ * logs the parent dir then throws an exception
+ * @param dir the directory
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException if it is not a directory
+ */
+ public static void verifyIsDir(File dir, Logger errorlog) throws
+ FileNotFoundException {
+ if (!dir.exists()) {
+ errorlog.warn("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(dir.toString());
+ }
+ if (!dir.isDirectory()) {
+ errorlog.info("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(
+ "Not a directory: " + dir);
+ }
+ }
+
+ /**
+ * Verify that a file exists
+ * @param file file
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException
+ */
+ public static void verifyFileExists(File file, Logger errorlog) throws
+ FileNotFoundException {
+ if (!file.exists()) {
+ errorlog.warn("contents of {}: {}", file,
+ listDir(file.getParentFile()));
+ throw new FileNotFoundException(file.toString());
+ }
+ if (!file.isFile()) {
+ throw new FileNotFoundException("Not a file: " + file.toString());
+ }
+ }
+
+ /**
+ * verify that a config option is set
+ * @param configuration config
+ * @param key key
+ * @return the value, in case it needs to be verified too
+ * @throws BadConfigException if the key is missing
+ */
+ public static String verifyOptionSet(Configuration configuration, String key,
+ boolean allowEmpty) throws BadConfigException {
+ String val = configuration.get(key);
+ if (val == null) {
+ throw new BadConfigException(
+ "Required configuration option \"%s\" not defined ", key);
+ }
+ if (!allowEmpty && val.isEmpty()) {
+ throw new BadConfigException(
+ "Configuration option \"%s\" must not be empty", key);
+ }
+ return val;
+ }
+
+ /**
+ * Verify that a keytab property is defined and refers to a non-empty file
+ *
+ * @param siteConf configuration
+ * @param prop property to look for
+ * @return the file referenced
+ * @throws BadConfigException on a failure
+ */
+ public static File verifyKeytabExists(Configuration siteConf,
+ String prop) throws
+ BadConfigException {
+ String keytab = siteConf.get(prop);
+ if (keytab == null) {
+ throw new BadConfigException("Missing keytab property %s",
+ prop);
+
+ }
+ File keytabFile = new File(keytab);
+ if (!keytabFile.exists()) {
+ throw new BadConfigException("Missing keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ if (keytabFile.length() == 0 || !keytabFile.isFile()) {
+ throw new BadConfigException("Invalid keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ return keytabFile;
+ }
+
+ /**
+ * Add a subpath to an existing URL. This extends
+ * the path, inserting a / between all entries
+ * if needed.
+ * @param base base path/URL
+ * @param path subpath
+ * @return base+"/"+subpath
+ */
+ public static String appendToURL(String base, String path) {
+ StringBuilder fullpath = new StringBuilder(base);
+ if (!base.endsWith("/")) {
+ fullpath.append("/");
+ }
+ if (path.startsWith("/")) {
+ fullpath.append(path.substring(1));
+ } else {
+ fullpath.append(path);
+ }
+ return fullpath.toString();
+ }
+
+ /**
+ * Truncate the given string to a maximum length provided
+ * with a pad (...) added to the end if expected size if more than 10.
+ * @param toTruncate string to truncate; may be null
+ * @param maxSize maximum size
+ * @return the truncated/padded string.
+ */
+ public static String truncate(String toTruncate, int maxSize) {
+ if (toTruncate == null || maxSize < 1
+ || toTruncate.length() <= maxSize) {
+ return toTruncate;
+ }
+
+ String pad = "...";
+ if (maxSize < 10) {
+ pad = "";
+ }
+ return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
+ }
+
+ /**
+ * Given a source folder create zipped file
+ *
+ * @param srcFolder
+ * @param zipFile
+ *
+ * @throws IOException
+ */
+ public static void zipFolder(File srcFolder, File zipFile) throws IOException {
+ log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true);
+
+ byte[] buffer = new byte[1024];
+
+ try (FileOutputStream fos = new FileOutputStream(zipFile)) {
+ try (ZipOutputStream zos = new ZipOutputStream(fos)) {
+
+ for (String file : files) {
+ ZipEntry ze = new ZipEntry(file);
+ zos.putNextEntry(ze);
+ try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ zos.write(buffer, 0, len);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Given a source folder create a tar.gz file
+ *
+ * @param libDirs
+ * @param tarGzipFile
+ *
+ * @throws IOException
+ */
+ public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
+ FilenameFilter filter) throws IOException {
+ log.info("Tar-gzipping folders {} to {}", libDirs,
+ tarGzipFile.getAbsolutePath());
+
+ try(TarArchiveOutputStream taos =
+ new TarArchiveOutputStream(new GZIPOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
+ for (String libDir : libDirs) {
+ File srcFolder = new File(libDir);
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true, filter);
+ for (String file : files) {
+ File srcFile = new File(srcFolder, file);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ srcFile, file);
+ taos.putArchiveEntry(tarEntry);
+ try(FileInputStream in = new FileInputStream(srcFile)) {
+ org.apache.commons.io.IOUtils.copy(in, taos);
+ }
+ taos.flush();
+ taos.closeArchiveEntry();
+ }
+ }
+ }
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative) {
+ generateFileList(fileList, node, rootFolder, relative, null);
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative, FilenameFilter filter) {
+ if (node.isFile()) {
+ String fileFullPath = node.toString();
+ if (relative) {
+ fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
+ fileFullPath.length()));
+ } else {
+ fileList.add(fileFullPath);
+ }
+ }
+
+ if (node.isDirectory()) {
+ String[] subNode = node.list(filter);
+ if (subNode == null || subNode.length == 0) {
+ return;
+ }
+ for (String filename : subNode) {
+ generateFileList(fileList, new File(node, filename), rootFolder,
+ relative, filter);
+ }
+ }
+ }
+
+ /**
+ * Check for any needed libraries being present. On Unix none are needed;
+ * on windows they must be present
+ * @return true if all is well
+ */
+ public static String checkForRequiredNativeLibraries() {
+
+ if (!Shell.WINDOWS) {
+ return "";
+ }
+ StringBuilder errorText = new StringBuilder("");
+ if (!NativeIO.isAvailable()) {
+ errorText.append("No native IO library. ");
+ }
+ try {
+ String path = Shell.getQualifiedBinPath(WINUTILS);
+ log.debug("winutils is at {}", path);
+ } catch (IOException e) {
+ errorText.append("No " + WINUTILS);
+ log.warn("No winutils: {}", e, e);
+ }
+ try {
+ File target = new File("target");
+ FileUtil.canRead(target);
+ } catch (UnsatisfiedLinkError e) {
+ log.warn("Failing to link to native IO methods: {}", e, e);
+ errorText.append("No native IO methods");
+ }
+ return errorText.toString();
+ }
+
+ /**
+ * Strictly verify that windows utils is present.
+ * Checks go as far as opening the file and looking for
+ * the headers.
+ * @throws IOException on any problem reading the file
+ * @throws FileNotFoundException if the file is not considered valid
+ */
+ public static void maybeVerifyWinUtilsValid() throws
+ IOException,
+ SliderException {
+ String errorText = SliderUtils.checkForRequiredNativeLibraries();
+ if (!errorText.isEmpty()) {
+ throw new BadClusterStateException(errorText);
+ }
+ }
+
+ /**
+ * Write bytes to a file
+ * @param outfile output file
+ * @param data data to write
+ * @throws IOException on any IO problem
+ */
+ public static void write(File outfile, byte[] data)
+ throws IOException {
+ File parentDir = outfile.getCanonicalFile().getParentFile();
+ if (parentDir == null) {
+ throw new IOException(outfile.getPath() + " has no parent dir");
+ }
+ if (!parentDir.exists()) {
+ if(!parentDir.mkdirs()) {
+ throw new IOException("Failed to create parent directory " + parentDir);
+ }
+ }
+ SliderUtils.verifyIsDir(parentDir, log);
+ try(FileOutputStream out = new FileOutputStream(outfile)) {
+ out.write(data);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first
+ * Specifically: the one whose start time value is greater.
+ */
+ private static class MostRecentlyStartedAppFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getStartTime();
+ long y = r2.getStartTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first.
+ * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
+ */
+ private static class MostRecentlyStartedOrFinishedFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long started1 = r1.getStartTime();
+ long started2 = r2.getStartTime();
+ long finished1 = r1.getFinishTime();
+ long finished2 = r2.getFinishTime();
+ long lastEvent1 = Math.max(started1, finished1);
+ long lastEvent2 = Math.max(started2, finished2);
+ return compareTwoLongsReverse(lastEvent1, lastEvent2);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recently finished app comes first
+ * Specifically: the one whose finish time value is greater.
+ */
+ private static class MostRecentAppFinishFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getFinishTime();
+ long y = r2.getFinishTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare two long values for sorting. As the return value for
+ * comparators must be int, the simple value of <code>x-y</code>
+ * is inapplicable
+ * @param x x value
+ * @param y y value
+ * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
+ */
+ public static int compareTwoLongsReverse(long x, long y) {
+ return (x < y) ? 1 : ((x == y) ? 0 : -1);
+ }
+
+ public static String createNameTag(String name) {
+ return "Name: " + name;
+ }
+
+ public static String createVersionTag(String version) {
+ return "Version: " + version;
+ }
+
+ public static String createDescriptionTag(String description) {
+ return "Description: " + description;
+ }
+
+ public static final String DAYS = ".days";
+ public static final String HOURS = ".hours";
+ public static final String MINUTES = ".minutes";
+ public static final String SECONDS = ".seconds";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
new file mode 100644
index 0000000..1fa07ce
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.net.HostAndPort;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ZookeeperUtils {
+ public static final int DEFAULT_PORT = 2181;
+
+ public static String buildConnectionString(String zkHosts, int port) {
+ String zkPort = Integer.toString(port);
+ //parse the hosts
+ String[] hostlist = zkHosts.split(",", 0);
+ String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
+ return quorum;
+ }
+
+ /**
+ * Take a quorum list and split it to (trimmed) pairs
+ * @param hostPortQuorumList list of form h1:port, h2:port2,...
+ * @return a possibly empty list of values between commas. They may not be
+ * valid hostname:port pairs
+ */
+ public static List<String> splitToPairs(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ int len = 0;
+ if (strings != null) {
+ len = strings.length;
+ }
+ List<String> tuples = new ArrayList<String>(len);
+ if (strings != null) {
+ for (String s : strings) {
+ tuples.add(s.trim());
+ }
+ }
+ return tuples;
+ }
+
+ /**
+ * Split a quorum list into a list of hostnames and ports
+ * @param hostPortQuorumList split to a list of hosts and ports
+ * @return a list of values
+ */
+ public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ int len = 0;
+ if (strings != null) {
+ len = strings.length;
+ }
+ List<HostAndPort> list = new ArrayList<HostAndPort>(len);
+ if (strings != null) {
+ for (String s : strings) {
+ list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
+ }
+ }
+ return list;
+ }
+
+ /**
+ * Build up to a hosts only list
+ * @param hostAndPorts
+ * @return a list of the hosts only
+ */
+ public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
+ StringBuilder sb = new StringBuilder();
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ sb.append(hostAndPort.getHostText()).append(",");
+ }
+ if (sb.length() > 0) {
+ sb.delete(sb.length() - 1, sb.length());
+ }
+ return sb.toString();
+ }
+
+ public static String buildQuorumEntry(HostAndPort hostAndPort,
+ int defaultPort) {
+ String s = hostAndPort.toString();
+ if (hostAndPort.hasPort()) {
+ return s;
+ } else {
+ return s + ":" + defaultPort;
+ }
+ }
+
+ /**
+ * Build a quorum list, injecting a ":defaultPort" ref if needed on
+ * any entry without one
+ * @param hostAndPorts
+ * @param defaultPort
+ * @return
+ */
+ public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
+ List<String> entries = new ArrayList<String>(hostAndPorts.size());
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ entries.add(buildQuorumEntry(hostAndPort, defaultPort));
+ }
+ return SliderUtils.join(entries, ",", false);
+ }
+
+ public static String convertToHostsOnlyList(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
+ }
+
+ public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts =
+ ZookeeperUtils.splitToHostsAndPorts(quorum);
+ if (hostAndPorts.isEmpty()) {
+ throw new BadConfigException("empty zookeeper quorum");
+ }
+ return hostAndPorts;
+ }
+
+ public static int getFirstPort(String quorum, int defVal) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ int port = hostAndPorts.get(0).getPortOrDefault(defVal);
+ return port;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
new file mode 100644
index 0000000..0a21c24
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.yarn.proto";
+option java_outer_classname = "ClientAMProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.yarn;
+
+service ClientAMProtocolService {
+ rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto);
+ rpc getStatus(GetStatusRequestProto) returns (GetStatusResponseProto);
+ rpc stop(StopRequestProto) returns (StopResponseProto);
+}
+
+message FlexComponentsRequestProto {
+ repeated ComponentCountProto components = 1;
+}
+
+message ComponentCountProto {
+ optional string name = 1;
+ optional int64 numberOfContainers = 2;
+}
+
+message FlexComponentsResponseProto{
+}
+
+message GetStatusRequestProto {
+
+}
+message GetStatusResponseProto {
+ optional string status = 1;
+}
+
+message StopRequestProto {
+
+}
+
+message StopResponseProto {
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
new file mode 100644
index 0000000..d343a03
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.base.Supplier;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.NMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.ComponentState;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+import static org.mockito.Mockito.mock;
+
+public class MockServiceAM extends ServiceMaster {
+
+ Service service;
+ // The list of containers fed by tests to be returned on
+ // AMRMClientCallBackHandler#onContainersAllocated
+ final List<Container> feedContainers =
+ Collections.synchronizedList(new LinkedList<>());
+
+ public MockServiceAM(Service service) {
+ super(service.getName());
+ this.service = service;
+ }
+
+
+ @Override
+ protected ContainerId getAMContainerId()
+ throws BadClusterStateException {
+ return ContainerId.newContainerId(ApplicationAttemptId
+ .newInstance(ApplicationId.fromString(service.getId()), 1), 1);
+ }
+
+ @Override
+ protected Path getAppDir() {
+ Path path = new Path(new Path("target", "apps"), service.getName());
+ System.out.println("Service path: " + path);
+ return path;
+ }
+
+ @Override
+ protected ServiceScheduler createServiceScheduler(ServiceContext context)
+ throws IOException, YarnException {
+ return new ServiceScheduler(context) {
+
+ @Override
+ protected YarnRegistryViewForProviders createYarnRegistryOperations(
+ ServiceContext context, RegistryOperations registryClient) {
+ return mock(YarnRegistryViewForProviders.class);
+ }
+
+ @Override
+ protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
+ AMRMClientImpl client1 = new AMRMClientImpl() {
+ @Override public AllocateResponse allocate(float progressIndicator)
+ throws YarnException, IOException {
+
+ AllocateResponse.AllocateResponseBuilder builder =
+ AllocateResponse.newBuilder();
+ synchronized (feedContainers) {
+ if (feedContainers.isEmpty()) {
+ System.out.println("Allocating........ no containers");
+ return builder.build();
+ } else {
+ // The AMRMClient will return containers for compoenent that are
+ // at FLEXING state
+ List<Container> allocatedContainers = new LinkedList<>();
+ Iterator<Container> itor = feedContainers.iterator();
+ while (itor.hasNext()) {
+ Container c = itor.next();
+ org.apache.hadoop.yarn.service.component.Component component =
+ componentsById.get(c.getAllocationRequestId());
+ if (component.getState() == ComponentState.FLEXING) {
+ System.out.println("Allocated container " + c.getId());
+ allocatedContainers.add(c);
+ itor.remove();
+ }
+ }
+ return builder.allocatedContainers(allocatedContainers).build();
+ }
+ }
+ }
+
+ @Override
+ public RegisterApplicationMasterResponse registerApplicationMaster(
+ String appHostName, int appHostPort, String appTrackingUrl) {
+ return mock(RegisterApplicationMasterResponse.class);
+ }
+
+ @Override public void unregisterApplicationMaster(
+ FinalApplicationStatus appStatus, String appMessage,
+ String appTrackingUrl) {
+ // DO nothing
+ }
+ };
+
+ return AMRMClientAsync
+ .createAMRMClientAsync(client1, 1000,
+ this.new AMRMClientCallback());
+ }
+
+ @Override
+ public NMClientAsync createNMClient() {
+ NMClientAsync nmClientAsync = super.createNMClient();
+ nmClientAsync.setClient(mock(NMClient.class));
+ return nmClientAsync;
+ }
+ };
+ }
+
+ @Override protected void loadApplicationJson(ServiceContext context,
+ SliderFileSystem fs) throws IOException {
+ context.service = service;
+ }
+
+ /**
+ *
+ * @param service The service for the component
+ * @param id The id for the container
+ * @param compName The component to which the container is fed
+ * @return
+ */
+ public Container feedContainerToComp(Service service, int id,
+ String compName) {
+ ApplicationId applicationId = ApplicationId.fromString(service.getId());
+ ContainerId containerId = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(applicationId, 1), id);
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+ Container container = Container
+ .newInstance(containerId, nodeId, "localhost",
+ Resource.newInstance(100, 1), Priority.newInstance(0), null);
+
+ long allocateId =
+ context.scheduler.getAllComponents().get(compName).getAllocateId();
+ container.setAllocationRequestId(allocateId);
+ synchronized (feedContainers) {
+ feedContainers.add(container);
+ }
+ return container;
+ }
+
+ public void flexComponent(String compName, long numberOfContainers)
+ throws IOException {
+ ClientAMProtocol.ComponentCountProto componentCountProto =
+ ClientAMProtocol.ComponentCountProto.newBuilder().setName(compName)
+ .setNumberOfContainers(numberOfContainers).build();
+ ClientAMProtocol.FlexComponentsRequestProto requestProto =
+ ClientAMProtocol.FlexComponentsRequestProto.newBuilder()
+ .addComponents(componentCountProto).build();
+ context.clientAMService.flexComponents(requestProto);
+ }
+
+ public Component getComponent(String compName) {
+ return context.scheduler.getAllComponents().get(compName);
+ }
+
+ public void waitForDependenciesSatisfied(String compName)
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override public Boolean get() {
+ return context.scheduler.getAllComponents().get(compName)
+ .areDependenciesReady();
+ }
+ }, 1000, 20000);
+ }
+
+ public void waitForNumDesiredContainers(String compName,
+ int numDesiredContainers) throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override public Boolean get() {
+ return context.scheduler.getAllComponents().get(compName)
+ .getNumDesiredInstances() == numDesiredContainers;
+ }
+ }, 1000, 20000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
new file mode 100644
index 0000000..0f4f598
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+
+public class ServiceTestUtils {
+
+ public static final JsonSerDeser<Service> JSON_SER_DESER =
+ new JsonSerDeser<>(Service.class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+
+ // Example service definition
+ // 2 components, each of which has 2 containers.
+ protected Service createExampleApplication() {
+ Service exampleApp = new Service();
+ exampleApp.setName("example-app");
+ exampleApp.addComponent(createComponent("compa"));
+ exampleApp.addComponent(createComponent("compb"));
+ return exampleApp;
+ }
+
+ protected Component createComponent(String name) {
+ return createComponent(name, 2L, "sleep 1000");
+ }
+
+ protected Component createComponent(String name, long numContainers,
+ String command) {
+ Component comp1 = new Component();
+ comp1.setNumberOfContainers(numContainers);
+ comp1.setLaunchCommand(command);
+ comp1.setName(name);
+ Resource resource = new Resource();
+ comp1.setResource(resource);
+ resource.setMemory("128");
+ resource.setCpus(1);
+ return comp1;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/58] [abbrv] hadoop git commit: Rebase onto latest trunk. minor
conflicts
Posted by ji...@apache.org.
Rebase onto latest trunk. minor conflicts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd96c4c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd96c4c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd96c4c2
Branch: refs/heads/yarn-native-services
Commit: bd96c4c235c15c1df6ba37544f8e15004ad1d09d
Parents: 59fe98c
Author: Jian He <ji...@apache.org>
Authored: Wed Aug 30 22:48:35 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:13 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/util/Apps.java | 2 ++
.../rmapp/attempt/RMAppAttemptImpl.java | 35 ++------------------
2 files changed, 4 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index 5efbd2e..685c6d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -150,6 +150,8 @@ public class Apps {
}
// Check if should black list the node based on container exit status
+ @Private
+ @Unstable
public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
switch (exitStatus) {
case ContainerExitStatus.PREEMPTED:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 4997bc6..d3fbcb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -107,6 +107,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BoundedAppender;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -1539,38 +1540,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
}
- private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
- switch (exitStatus) {
- case ContainerExitStatus.PREEMPTED:
- case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER:
- case ContainerExitStatus.KILLED_BY_APPMASTER:
- case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION:
- case ContainerExitStatus.ABORTED:
- // Neither the app's fault nor the system's fault. This happens by design,
- // so no need for skipping nodes
- return false;
- case ContainerExitStatus.DISKS_FAILED:
- // This container is marked with this exit-status means that the node is
- // already marked as unhealthy given that most of the disks failed. So, no
- // need for any explicit skipping of nodes.
- return false;
- case ContainerExitStatus.KILLED_EXCEEDED_VMEM:
- case ContainerExitStatus.KILLED_EXCEEDED_PMEM:
- // No point in skipping the node as it's not the system's fault
- return false;
- case ContainerExitStatus.SUCCESS:
- return false;
- case ContainerExitStatus.INVALID:
- // Ideally, this shouldn't be considered for skipping a node. But in
- // reality, it seems like there are cases where we are not setting
- // exit-code correctly and so it's better to be conservative. See
- // YARN-4284.
- return true;
- default:
- return true;
- }
- }
-
private static final class UnmanagedAMAttemptSavedTransition
extends AMLaunchedTransition {
@Override
@@ -1954,7 +1923,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
containerFinishedEvent.getContainerStatus();
if (containerStatus != null) {
int exitStatus = containerStatus.getExitStatus();
- if (shouldCountTowardsNodeBlacklisting(exitStatus)) {
+ if (Apps.shouldCountTowardsNodeBlacklisting(exitStatus)) {
appAttempt.addAMNodeToBlackList(nodeId);
}
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/58] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in
yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
new file mode 100644
index 0000000..9022268
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -0,0 +1,592 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN Service API
+
+Bringing a new service on YARN today is not a simple experience. The APIs of existing
+frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs)
+or writing a complex spec (for declarative frameworks).
+
+This simplified REST API can be used to create and manage the lifecycle of YARN services.
+In most cases, the application owner will not be forced to make any changes to their applications.
+This is primarily true if the application is packaged with containerization technologies like Docker.
+
+This document describes the API specifications (aka. YarnFile) for deploying/managing
+containerized services on YARN. The same JSON spec can be used for both REST API
+and CLI to manage the services.
+
+
+### Version information
+Version: 1.0.0
+
+### License information
+License: Apache 2.0
+License URL: http://www.apache.org/licenses/LICENSE-2.0.html
+
+### URI scheme
+Host: host.mycompany.com
+
+Port: 9191(default)
+
+Schemes: HTTP
+
+### Consumes
+
+* application/json
+
+
+### Produces
+
+* application/json
+
+
+## Paths
+### Create a service
+```
+POST /ws/v1/services
+```
+
+#### Description
+
+Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|BodyParameter|Service|Service request object|true|Service||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|202|The request to create a service is accepted|No Content|
+|400|Invalid service definition provided in the request body|No Content|
+|500|Failed to create a service|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### (TBD) List of services running in the cluster.
+```
+GET /ws/v1/services
+```
+
+#### Description
+
+Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|An array of services|Service array|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get current version of the API server.
+```
+GET /ws/v1/services/version
+```
+
+#### Description
+
+Get current version of the API server.
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|Successful request|No Content|
+
+
+### Update a service or upgrade the binary version of the components of a running service
+```
+PUT /ws/v1/services/{service_name}
+```
+
+#### Description
+
+Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+|BodyParameter|Service|The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service|true|Service||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|204|Update or upgrade was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Destroy a service
+```
+DELETE /ws/v1/services/{service_name}
+```
+
+#### Description
+
+Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|204|Destroy was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get details of a service.
+```
+GET /ws/v1/services/{service_name}
+```
+
+#### Description
+
+Return the details (including containers) of a running service
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|a service object|object|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Flex a component's number of instances.
+```
+PUT /ws/v1/services/{service_name}/components/{component_name}
+```
+
+#### Description
+
+Set a component's desired number of instanes
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+|PathParameter|component_name|Component name|true|string||
+|BodyParameter|Component|The definition of a component which contains the updated number of instances.|true|Component||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|Flex was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+## Definitions
+### Artifact
+
+Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|id|Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.|true|string||
+|type|Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).|false|enum (DOCKER, TARBALL, SERVICE)|DOCKER|
+|uri|Artifact location to support multiple artifact stores (optional).|false|string||
+
+
+### Component
+
+One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|name|Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.|true|string||
+|dependencies|An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.|false|string array||
+|readiness_check|Readiness check for this component.|false|ReadinessCheck||
+|artifact|Artifact of the component (optional). If not specified, the service level global artifact takes effect.|false|Artifact||
+|launch_command|The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).|false|string||
+|resource|Resource of this component (optional). If not specified, the service level global resource takes effect.|false|Resource||
+|number_of_containers|Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.|false|integer (int64)||
+|run_privileged_container|Run all containers of this component in privileged mode (YARN-4262).|false|boolean||
+|placement_policy|Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.|false|PlacementPolicy||
+|configuration|Config properties for this component.|false|Configuration||
+|quicklinks|A list of quicklink keys defined at the service level, and to be resolved by this component.|false|string array||
+
+
+### ConfigFile
+
+A config file that needs to be created and made available as a volume in a service component container.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|type|Config file in the standard format like xml, properties, json, yaml, template.|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, ENV, HADOOP_XML)||
+|dest_file|The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.|false|string||
+|src_file|This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.|false|string||
+|props|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
+
+
+### Configuration
+
+Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|properties|A blob of key-value pairs of common service properties.|false|object||
+|env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object||
+|files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array||
+
+
+### Container
+
+An instance of a running service container.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|id|Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.|false|string||
+|launch_time|The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.|false|string (date)||
+|ip|IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
+|hostname|Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
+|bare_host|The bare node or host in which the container is running, e.g. cn008.example.com.|false|string||
+|state|State of the container of a service.|false|ContainerState||
+|component_name|Name of the component that this container instance belongs to.|false|string||
+|resource|Resource used for this container.|false|Resource||
+|artifact|Artifact used for this container.|false|Artifact||
+|privileged_container|Container running in privileged mode or not.|false|boolean||
+
+
+### ContainerState
+
+The current state of the container of a service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|state|enum of the state of the container|false|enum (INIT, STARTED, READY)||
+
+
+### PlacementPolicy
+
+Placement policy of an instance of a service. This feature is in the works in YARN-6592.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|label|Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.|false|string||
+
+
+### ReadinessCheck
+
+A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|type|E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).|true|enum (HTTP, PORT)||
+|props|A blob of key value pairs that will be used to configure the check.|false|object||
+|artifact|Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET|false|Artifact||
+
+
+### Resource
+
+Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string||
+|cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)||
+|memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string||
+
+
+### Service
+
+a service resource has the following attributes.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|name|A unique service name. If Registry DNS is enabled, the max length is 63 characters.|true|string||
+|id|A unique service id.|false|string||
+|artifact|Artifact of single-component service.|false|Artifact||
+|resource|Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.|false|Resource||
+|launch_command|The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.|false|string||
+|launch_time|The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.|false|string (date)||
+|number_of_containers|Number of containers for each component in the service. Each component can further override this service-level global default.|false|integer (int64)||
+|number_of_running_containers|In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.|false|integer (int64)||
+|lifetime|Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.|false|integer (int64)||
+|placement_policy|(TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.|false|PlacementPolicy||
+|components|Components of a service.|false|Component array||
+|configuration|Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.|false|Configuration||
+|containers|Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.|false|Container array||
+|state|State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.|false|ServiceState||
+|quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
+|queue|The YARN queue that this service should be submitted to.|false|string||
+
+
+### ServiceState
+
+The current state of a service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, READY, STOPPED, FAILED)||
+
+
+### ServiceStatus
+
+The current status of a submitted service, returned as a response to the GET API.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|diagnostics|Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.|false|string||
+|state|Service state.|false|ServiceState||
+|code|An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.|false|integer (int32)||
+
+
+
+## Examples
+
+### Create a simple single-component service with most attribute values as defaults
+POST URL - http://localhost:9191/ws/v1/services
+
+##### POST Request JSON
+```json
+{
+ "name": "hello-world",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "nginx:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "./start_nginx.sh",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+
+##### GET Response JSON
+GET URL - http://localhost:9191/ws/v1/services/hello-world
+
+Note, lifetime value of -1 means unlimited lifetime.
+
+```json
+{
+ "name": "hello-world",
+ "id": "application_1503963985568_0002",
+ "lifetime": -1,
+ "components": [
+ {
+ "name": "hello",
+ "dependencies": [],
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": [],
+ "containers": [
+ {
+ "id": "container_e03_1503963985568_0002_01_000001",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051512412,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-0"
+ },
+ {
+ "id": "container_e03_1503963985568_0002_01_000002",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051536450,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-1"
+ }
+ ],
+ "launch_command": "./start_nginx.sh",
+ "number_of_containers": 1,
+ "run_privileged_container": false
+ }
+ ],
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": {}
+}
+
+```
+### Update to modify the lifetime of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+
+Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
+
+```json
+{
+ "lifetime": 3600
+}
+```
+### Stop a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STOPPED"
+}
+```
+
+### Start a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STARTED"
+}
+```
+
+### Update to flex up/down the no of containers (instances) of a component of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
+
+##### PUT Request JSON
+```json
+{
+ "name": "hello",
+ "number_of_containers": 3
+}
+```
+
+### Destroy a service
+DELETE URL - http://localhost:9191/ws/v1/services/hello-world
+
+***
+
+### Create a complicated service - HBase
+POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
+
+##### POST Request JSON
+
+```json
+{
+ "name": "hbase-app-1",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "hbasemaster",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "regionserver",
+ "number_of_containers": 3,
+ "unique_component_support": "true",
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010",
+ "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
+ "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
+ }
+}
+```
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
new file mode 100644
index 0000000..7baa284
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Build a single command line to include in the container commands;
+ * Special support for JVM command buildup.
+ */
+public class CommandLineBuilder {
+ protected final List<String> argumentList = new ArrayList<>(20);
+
+ /**
+ * Add an entry to the command list
+ * @param args arguments -these will be converted strings
+ */
+ public void add(Object... args) {
+ for (Object arg : args) {
+ argumentList.add(arg.toString());
+ }
+ }
+
+ // Get the number of arguments
+ public int size() {
+ return argumentList.size();
+ }
+
+ /**
+ * Append the output and error files to the tail of the command
+ * @param stdout out
+ * @param stderr error. Set this to null to append into stdout
+ */
+ public void addOutAndErrFiles(String stdout, String stderr) {
+ Preconditions.checkNotNull(stdout, "Null output file");
+ Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
+ // write out the path output
+ argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stdout);
+ if (stderr != null) {
+ argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stderr);
+ } else {
+ argumentList.add("2>&1");
+ }
+ }
+
+ /**
+ * This just returns the command line
+ * @see #build()
+ * @return the command line
+ */
+ @Override
+ public String toString() {
+ return build();
+ }
+
+ /**
+ * Build the command line
+ * @return the command line
+ */
+ public String build() {
+ return SliderUtils.join(argumentList, " ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
new file mode 100644
index 0000000..0e51a62
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class ContainerLaunchService extends AbstractService{
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ContainerLaunchService.class);
+
+ private ExecutorService executorService;
+ private SliderFileSystem fs;
+
+ public ContainerLaunchService(SliderFileSystem fs) {
+ super(ContainerLaunchService.class.getName());
+ this.fs = fs;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ executorService = Executors.newCachedThreadPool();
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+ super.serviceStop();
+ }
+
+ public void launchCompInstance(Service service,
+ ComponentInstance instance, Container container) {
+ ContainerLauncher launcher =
+ new ContainerLauncher(service, instance, container);
+ executorService.execute(launcher);
+ }
+
+ private class ContainerLauncher implements Runnable {
+ public final Container container;
+ public final Service service;
+ public ComponentInstance instance;
+
+ public ContainerLauncher(
+ Service service,
+ ComponentInstance instance, Container container) {
+ this.container = container;
+ this.service = service;
+ this.instance = instance;
+ }
+
+ @Override public void run() {
+ Component compSpec = instance.getCompSpec();
+ ProviderService provider = ProviderFactory.getProviderService(
+ compSpec.getArtifact());
+ AbstractLauncher launcher = new AbstractLauncher(fs, null);
+ try {
+ provider.buildContainerLaunchContext(launcher, service,
+ instance, fs, getConfig());
+ instance.getComponent().getScheduler().getNmClient()
+ .startContainerAsync(container,
+ launcher.completeContainerLaunch());
+ } catch (Exception e) {
+ LOG.error(instance.getCompInstanceId()
+ + ": Failed to launch container. ", e);
+
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
new file mode 100644
index 0000000..fce58e5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.text.DateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+
+/**
+ * Utils to work with credentials and tokens.
+ *
+ * Designed to be movable to Hadoop core
+ */
+public final class CredentialUtils {
+
+ private CredentialUtils() {
+ }
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CredentialUtils.class);
+
+ /**
+ * Save credentials to a byte buffer. Returns null if there were no
+ * credentials to save
+ * @param credentials credential set
+ * @return a byte buffer of serialized tokens
+ * @throws IOException if the credentials could not be written to the stream
+ */
+ public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException {
+ ByteBuffer buffer = null;
+ if (!credentials.getAllTokens().isEmpty()) {
+ DataOutputBuffer dob = new DataOutputBuffer();
+ try {
+ credentials.writeTokenStorageToStream(dob);
+ } finally {
+ dob.close();
+ }
+ buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ }
+ return buffer;
+ }
+
+ /**
+ * Save credentials to a file
+ * @param file file to save to (will be overwritten)
+ * @param credentials credentials to write
+ * @throws IOException
+ */
+ public static void saveTokens(File file,
+ Credentials credentials) throws IOException {
+ try(DataOutputStream daos = new DataOutputStream(
+ new FileOutputStream(file))) {
+ credentials.writeTokenStorageToStream(daos);
+ }
+ }
+
+ /**
+ * Look up and return the resource manager's principal. This method
+ * automatically does the <code>_HOST</code> replacement in the principal and
+ * correctly handles HA resource manager configurations.
+ *
+ * From: YARN-4629
+ * @param conf the {@link Configuration} file from which to read the
+ * principal
+ * @return the resource manager's principal string
+ * @throws IOException thrown if there's an error replacing the host name
+ */
+ public static String getRMPrincipal(Configuration conf) throws IOException {
+ String principal = conf.get(RM_PRINCIPAL, "");
+ String hostname;
+ Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);
+
+ if (HAUtil.isHAEnabled(conf)) {
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ if (yarnConf.get(RM_HA_ID) == null) {
+ // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
+ // Any valid RM HA ID should work.
+ String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
+ Preconditions.checkState((rmIds != null) && (rmIds.length > 0),
+ "Not set " + RM_HA_IDS);
+ yarnConf.set(RM_HA_ID, rmIds[0]);
+ }
+
+ hostname = yarnConf.getSocketAddr(
+ RM_ADDRESS,
+ DEFAULT_RM_ADDRESS,
+ DEFAULT_RM_PORT).getHostName();
+ } else {
+ hostname = conf.getSocketAddr(
+ RM_ADDRESS,
+ DEFAULT_RM_ADDRESS,
+ DEFAULT_RM_PORT).getHostName();
+ }
+ return SecurityUtil.getServerPrincipal(principal, hostname);
+ }
+
+ /**
+ * Create and add any filesystem delegation tokens with
+ * the RM(s) configured to be able to renew them. Returns null
+ * on an insecure cluster (i.e. harmless)
+ * @param conf configuration
+ * @param fs filesystem
+ * @param credentials credentials to update
+ * @return a list of all added tokens.
+ * @throws IOException
+ */
+ public static Token<?>[] addRMRenewableFSDelegationTokens(Configuration conf,
+ FileSystem fs,
+ Credentials credentials) throws IOException {
+ Preconditions.checkArgument(conf != null);
+ Preconditions.checkArgument(credentials != null);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf),
+ credentials);
+ }
+ return null;
+ }
+
+ /**
+ * Add an FS delegation token which can be renewed by the current user
+ * @param fs filesystem
+ * @param credentials credentials to update
+ * @throws IOException problems.
+ */
+ public static void addSelfRenewableFSDelegationTokens(
+ FileSystem fs,
+ Credentials credentials) throws IOException {
+ Preconditions.checkArgument(fs != null);
+ Preconditions.checkArgument(credentials != null);
+ fs.addDelegationTokens(
+ getSelfRenewer(),
+ credentials);
+ }
+
+ public static String getSelfRenewer() throws IOException {
+ return UserGroupInformation.getLoginUser().getShortUserName();
+ }
+
+ /**
+ * Create and add an RM delegation token to the credentials
+ * @param yarnClient Yarn Client
+ * @param credentials to add token to
+ * @return the token which was added
+ * @throws IOException
+ * @throws YarnException
+ */
+ public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient,
+ Credentials credentials)
+ throws IOException, YarnException {
+ Configuration conf = yarnClient.getConfig();
+ Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf));
+ Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
+ Token<TokenIdentifier> rmDelegationToken =
+ ConverterUtils.convertFromYarn(
+ yarnClient.getRMDelegationToken(rmPrincipal),
+ rmDTService);
+ credentials.addToken(rmDelegationToken.getService(), rmDelegationToken);
+ return rmDelegationToken;
+ }
+
+ public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(
+ Configuration conf,
+ Credentials credentials)
+ throws IOException, YarnException {
+ if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
+ LOG.debug("Timeline service enabled -fetching token");
+
+ try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
+ timelineClient.init(conf);
+ timelineClient.start();
+ Token<TimelineDelegationTokenIdentifier> token =
+ timelineClient.getDelegationToken(
+ CredentialUtils.getRMPrincipal(conf));
+ credentials.addToken(token.getService(), token);
+ return token;
+ }
+ } else {
+ LOG.debug("Timeline service is disabled");
+ return null;
+ }
+ }
+
+ /**
+ * Filter a list of tokens from a set of credentials
+ * @param credentials credential source (a new credential set os re
+ * @param filter List of tokens to strip out
+ * @return a new, filtered, set of credentials
+ */
+ public static Credentials filterTokens(Credentials credentials,
+ List<Text> filter) {
+ Credentials result = new Credentials(credentials);
+ Iterator<Token<? extends TokenIdentifier>> iter =
+ result.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<? extends TokenIdentifier> token = iter.next();
+ LOG.debug("Token {}", token.getKind());
+ if (filter.contains(token.getKind())) {
+ LOG.debug("Filtering token {}", token.getKind());
+ iter.remove();
+ }
+ }
+ return result;
+ }
+
+ public static String dumpTokens(Credentials credentials, String separator) {
+ ArrayList<Token<? extends TokenIdentifier>> sorted =
+ new ArrayList<>(credentials.getAllTokens());
+ Collections.sort(sorted, new TokenComparator());
+ StringBuilder buffer = new StringBuilder(sorted.size()* 128);
+ for (Token<? extends TokenIdentifier> token : sorted) {
+ buffer.append(tokenToString(token)).append(separator);
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Create a string for people to look at
+ * @param token token to convert to a string form
+ * @return a printable view of the token
+ */
+ public static String tokenToString(Token<? extends TokenIdentifier> token) {
+ DateFormat df = DateFormat.getDateTimeInstance(
+ DateFormat.SHORT, DateFormat.SHORT);
+ StringBuilder buffer = new StringBuilder(128);
+ buffer.append(token.toString());
+ try {
+ TokenIdentifier ti = token.decodeIdentifier();
+ buffer.append("; ").append(ti);
+ if (ti instanceof AbstractDelegationTokenIdentifier) {
+ // details in human readable form, and compensate for information HDFS DT omits
+ AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti;
+ buffer.append("; Renewer: ").append(dt.getRenewer());
+ buffer.append("; Issued: ")
+ .append(df.format(new Date(dt.getIssueDate())));
+ buffer.append("; Max Date: ")
+ .append(df.format(new Date(dt.getMaxDate())));
+ }
+ } catch (IOException e) {
+ //marshall problem; not ours
+ LOG.debug("Failed to decode {}: {}", token, e, e);
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Get the expiry time of a token.
+ * @param token token to examine
+ * @return the time in milliseconds after which the token is invalid.
+ * @throws IOException
+ */
+ public static long getTokenExpiryTime(Token token) throws IOException {
+ TokenIdentifier identifier = token.decodeIdentifier();
+ Preconditions.checkState(identifier instanceof AbstractDelegationTokenIdentifier,
+ "Token %s of type: %s has an identifier which cannot be examined: %s",
+ token, token.getClass(), identifier);
+ AbstractDelegationTokenIdentifier id =
+ (AbstractDelegationTokenIdentifier) identifier;
+ return id.getMaxDate();
+ }
+
+ private static class TokenComparator
+ implements Comparator<Token<? extends TokenIdentifier>>, Serializable {
+ @Override
+ public int compare(Token<? extends TokenIdentifier> left,
+ Token<? extends TokenIdentifier> right) {
+ return left.getKind().toString().compareTo(right.getKind().toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
new file mode 100644
index 0000000..cbcb0d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+
+import java.util.Map;
+
+/**
+ * Command line builder purely for the Java CLI.
+ * Some of the <code>define</code> methods are designed to work with Hadoop tool and
+ * Slider launcher applications.
+ */
+public class JavaCommandLineBuilder extends CommandLineBuilder {
+
+ public JavaCommandLineBuilder() {
+ add(getJavaBinary());
+ }
+
+ /**
+ * Get the java binary. This is called in the constructor so don't try and
+ * do anything other than return a constant.
+ * @return the path to the Java binary
+ */
+ protected String getJavaBinary() {
+ return ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java";
+ }
+
+ /**
+ * Set the size of the heap if a non-empty heap is passed in.
+ * @param heap empty string or something like "128M" ,"1G" etc. The value is
+ * trimmed.
+ */
+ public void setJVMHeap(String heap) {
+ if (SliderUtils.isSet(heap)) {
+ add("-Xmx" + heap.trim());
+ }
+ }
+
+ /**
+ * Turn Java assertions on
+ */
+ public void enableJavaAssertions() {
+ add("-ea");
+ add("-esa");
+ }
+
+ /**
+ * Add a system property definition -must be used before setting the main entry point
+ * @param property
+ * @param value
+ */
+ public void sysprop(String property, String value) {
+ Preconditions.checkArgument(property != null, "null property name");
+ Preconditions.checkArgument(value != null, "null value");
+ add("-D" + property + "=" + value);
+ }
+
+ public JavaCommandLineBuilder forceIPv4() {
+ sysprop("java.net.preferIPv4Stack", "true");
+ return this;
+ }
+
+ public JavaCommandLineBuilder headless() {
+ sysprop("java.awt.headless", "true");
+ return this;
+ }
+
+ public boolean addConfOption(Configuration conf, String key) {
+ return defineIfSet(key, conf.get(key));
+ }
+
+ /**
+ * Add a varargs list of configuration parameters —if they are present
+ * @param conf configuration source
+ * @param keys keys
+ */
+ public void addConfOptions(Configuration conf, String... keys) {
+ for (String key : keys) {
+ addConfOption(conf, key);
+ }
+ }
+
+ /**
+ * Add all configuration options which match the prefix
+ * @param conf configuration
+ * @param prefix prefix, e.g {@code "slider."}
+ * @return the number of entries copied
+ */
+ public int addPrefixedConfOptions(Configuration conf, String prefix) {
+ int copied = 0;
+ for (Map.Entry<String, String> entry : conf) {
+ if (entry.getKey().startsWith(prefix)) {
+ define(entry.getKey(), entry.getValue());
+ copied++;
+ }
+ }
+ return copied;
+ }
+
+ /**
+ * Ass a configuration option to the command line of the application
+ * @param conf configuration
+ * @param key key
+ * @param defVal default value
+ * @return the resolved configuration option
+ * @throws IllegalArgumentException if key is null or the looked up value
+ * is null (that is: the argument is missing and devVal was null.
+ */
+ public String addConfOptionToCLI(Configuration conf,
+ String key,
+ String defVal) {
+ Preconditions.checkArgument(key != null, "null key");
+ String val = conf.get(key, defVal);
+ define(key, val);
+ return val;
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
+ * @param key key
+ * @param val value
+ * @throws IllegalArgumentException if either argument is null
+ */
+ public void define(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ Preconditions.checkArgument(val != null, "null value");
+ add("-D", key + "=" + val);
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
+ * is not null
+ * @param key key
+ * @param val value
+ */
+ public boolean defineIfSet(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ if (val != null) {
+ define(key, val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Add a mandatory config option
+ * @param conf configuration
+ * @param key key
+ * @throws BadConfigException if the key is missing
+ */
+ public void addMandatoryConfOption(Configuration conf,
+ String key) throws BadConfigException {
+ if (!addConfOption(conf, key)) {
+ throw new BadConfigException("Missing configuration option: " + key);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
new file mode 100644
index 0000000..db9de7a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+
+/**
+ * The system is in a bad state
+ */
+public class BadClusterStateException extends SliderException {
+ public BadClusterStateException(String message,
+ Object... args) {
+ super(EXIT_BAD_STATE, message, args);
+ }
+
+ public BadClusterStateException(Throwable throwable,
+ String message, Object... args) {
+ super(EXIT_BAD_STATE, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
new file mode 100644
index 0000000..41e3251
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+public class BadCommandArgumentsException extends SliderException {
+ public BadCommandArgumentsException(String s, Object... args) {
+ super(EXIT_COMMAND_ARGUMENT_ERROR, s, args);
+ }
+
+ public BadCommandArgumentsException(Throwable throwable, String message,
+ Object... args) {
+ super(EXIT_COMMAND_ARGUMENT_ERROR, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
new file mode 100644
index 0000000..8199c3c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/**
+ * An exception to raise on a bad configuration
+ */
+public class BadConfigException extends SliderException {
+
+ public BadConfigException(String s) {
+ super(EXIT_BAD_CONFIGURATION, s);
+ }
+
+ public BadConfigException(String message, Object... args) {
+ super(EXIT_BAD_CONFIGURATION, message, args);
+ }
+
+ public BadConfigException(
+ Throwable throwable,
+ String message, Object... args) {
+ super(EXIT_BAD_CONFIGURATION, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
new file mode 100644
index 0000000..83658c8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+public interface ErrorStrings {
+
+ String PRINTF_E_INSTANCE_ALREADY_EXISTS = "Service Instance \"%s\" already exists and is defined in %s";
+ String PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS = "Service Instance dir already exists: %s";
+
+ /**
+ * ERROR Strings
+ */
+ String ERROR_NO_ACTION = "No action specified";
+ String ERROR_UNKNOWN_ACTION = "Unknown command: ";
+ String ERROR_NOT_ENOUGH_ARGUMENTS =
+ "Not enough arguments for action: ";
+ String ERROR_PARSE_FAILURE =
+ "Failed to parse ";
+ /**
+ * All the remaining values after argument processing
+ */
+ String ERROR_TOO_MANY_ARGUMENTS =
+ "Too many arguments";
+ String ERROR_DUPLICATE_ENTRY = "Duplicate entry for ";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
new file mode 100644
index 0000000..d66b860
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/**
+ * Get the exit code of an exception. Making it an interface allows
+ * us to retrofit exit codes onto existing classes
+ */
+public interface ExitCodeProvider {
+
+ /**
+ * Method to get the exit code
+ * @return the exit code
+ */
+ int getExitCode();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
new file mode 100644
index 0000000..483fb48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/*
+ * Common Exit codes
+ * <p>
+ * Exit codes from 64 up are service specific.
+ * <p>
+ * Many of the exit codes are designed to resemble HTTP error codes,
+ * squashed into a single byte. e.g 44 , "not found" is the equivalent
+ * of 404
+ * <pre>
+ * 0-10: general command issues
+ * 30-39: equivalent to the 3XX responses, where those responses are
+ * considered errors by the service.
+ * 40-49: request-related errors
+ * 50-59: server-side problems. These may be triggered by the request.
+ * 64- : service specific error codes
+ * </pre>
+ */
+public interface LauncherExitCodes {
+
+ /**
+ * 0: success
+ */
+ int EXIT_SUCCESS = 0;
+
+ /**
+ * -1: generic "false" response. The operation worked but
+ * the result was not true
+ */
+ int EXIT_FALSE = -1;
+
+ /**
+ * Exit code when a client requested service termination: {@value}
+ */
+ int EXIT_CLIENT_INITIATED_SHUTDOWN = 1;
+
+ /**
+ * Exit code when targets could not be launched: {@value}
+ */
+ int EXIT_TASK_LAUNCH_FAILURE = 2;
+
+ /**
+ * Exit code when a control-C, kill -3, signal was picked up: {@value}
+ */
+ int EXIT_INTERRUPTED = 3;
+
+ /**
+ * Exit code when a usage message was printed: {@value}
+ */
+ int EXIT_USAGE = 4;
+
+ /**
+ * Exit code when something happened but we can't be specific: {@value}
+ */
+ int EXIT_OTHER_FAILURE = 5;
+
+ /**
+ * Exit code on connectivity problems: {@value}
+ */
+ int EXIT_MOVED = 31;
+
+ /**
+ * found: {@value}.
+ * <p>
+ * This is low value as in HTTP it is normally a success/redirect;
+ * whereas on the command line 0 is the sole success code.
+ * <p>
+ * <code>302 Found</code>
+ */
+ int EXIT_FOUND = 32;
+
+ /**
+ * Exit code on a request where the destination has not changed
+ * and (somehow) the command specified that this is an error.
+ * That is, this exit code is somehow different from a "success"
+ * : {@value}
+ * <p>
+ * <code>304 Not Modified </code>
+ */
+ int EXIT_NOT_MODIFIED = 34;
+
+ /**
+ * Exit code when the command line doesn't parse: {@value}, or
+ * when it is otherwise invalid.
+ * <p>
+ * <code>400 BAD REQUEST</code>
+ */
+ int EXIT_COMMAND_ARGUMENT_ERROR = 40;
+
+ /**
+ * The request requires user authentication: {@value}
+ * <p>
+ * <code>401 Unauthorized</code>
+ */
+ int EXIT_UNAUTHORIZED = 41;
+
+ /**
+ * Forbidden action: {@value}
+ * <p>
+ * <code>403: Forbidden</code>
+ */
+ int EXIT_FORBIDDEN = 43;
+
+ /**
+ * Something was not found: {@value}
+ * <p>
+ * <code>404: NOT FOUND</code>
+ */
+ int EXIT_NOT_FOUND = 44;
+
+ /**
+ * The operation is not allowed: {@value}
+ * <p>
+ * <code>405: NOT ALLOWED</code>
+ */
+ int EXIT_OPERATION_NOT_ALLOWED = 45;
+
+ /**
+ * The command is somehow not acceptable: {@value}
+ * <p>
+ * <code>406: NOT ACCEPTABLE</code>
+ */
+ int EXIT_NOT_ACCEPTABLE = 46;
+
+ /**
+ * Exit code on connectivity problems: {@value}
+ * <p>
+ * <code>408: Request Timeout</code>
+ */
+ int EXIT_CONNECTIVITY_PROBLEM = 48;
+
+ /**
+ * The request could not be completed due to a conflict with the current
+ * state of the resource. {@value}
+ * <p>
+ * <code>409: conflict</code>
+ */
+ int EXIT_CONFLICT = 49;
+
+ /**
+ * internal error: {@value}
+ * <p>
+ * <code>500 Internal Server Error</code>
+ */
+ int EXIT_INTERNAL_ERROR = 50;
+
+ /**
+ * Unimplemented feature: {@value}
+ * <p>
+ * <code>501: Not Implemented</code>
+ */
+ int EXIT_UNIMPLEMENTED = 51;
+
+ /**
+ * Service Unavailable; it may be available later: {@value}
+ * <p>
+ * <code>503 Service Unavailable</code>
+ */
+ int EXIT_SERVICE_UNAVAILABLE = 53;
+
+ /**
+ * The service does not support, or refuses to support this version: {@value}.
+ * If raised, this is expected to be raised server-side and likely due
+ * to client/server version incompatibilities.
+ * <p>
+ * <code> 505: Version Not Supported</code>
+ */
+ int EXIT_UNSUPPORTED_VERSION = 55;
+
+ /**
+ * Exit code when an exception was thrown from the service: {@value}
+ * <p>
+ * <code>5XX</code>
+ */
+ int EXIT_EXCEPTION_THROWN = 56;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
new file mode 100644
index 0000000..ef22b57
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+public interface RestApiErrorMessages {
+ String ERROR_APPLICATION_NAME_INVALID =
+ "Service name is either empty or not provided";
+ String ERROR_APPLICATION_NAME_INVALID_FORMAT =
+ "Service name %s is not valid - only lower case letters, digits, " +
+ "and hyphen are allowed, and the name must be no more " +
+ "than 63 characters";
+ String ERROR_COMPONENT_NAME_INVALID =
+ "Component name must be no more than %s characters: %s";
+ String ERROR_USER_NAME_INVALID =
+ "User name must be no more than 63 characters";
+
+ String ERROR_APPLICATION_NOT_RUNNING = "Service not running";
+ String ERROR_APPLICATION_DOES_NOT_EXIST = "Service not found";
+ String ERROR_APPLICATION_IN_USE = "Service already exists in started"
+ + " state";
+ String ERROR_APPLICATION_INSTANCE_EXISTS = "Service already exists in"
+ + " stopped/failed state (either restart with PUT or destroy with DELETE"
+ + " before creating a new one)";
+
+ String ERROR_SUFFIX_FOR_COMPONENT =
+ " for component %s (nor at the global level)";
+ String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
+ String ERROR_ARTIFACT_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_ARTIFACT_ID_INVALID =
+ "Artifact id (like docker image name) is either empty or not provided";
+ String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+ String ERROR_RESOURCE_INVALID = "Resource is not provided";
+ String ERROR_RESOURCE_FOR_COMP_INVALID =
+ ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_MEMORY_INVALID =
+ "Service resource or memory not provided";
+ String ERROR_RESOURCE_CPUS_INVALID =
+ "Service resource or cpus not provided";
+ String ERROR_RESOURCE_CPUS_INVALID_RANGE =
+ "Unacceptable no of cpus specified, either zero or negative";
+ String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
+ ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
+ ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
+ ERROR_RESOURCE_CPUS_INVALID_RANGE
+ + " for component %s (or at the global level)";
+ String ERROR_CONTAINERS_COUNT_INVALID =
+ "Invalid no of containers specified";
+ String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
+ ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_DEPENDENCY_INVALID = "Dependency %s for component %s is " +
+ "invalid, does not exist as a component";
+ String ERROR_DEPENDENCY_CYCLE = "Invalid dependencies, a cycle may " +
+ "exist: %s";
+
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
+ "Cannot specify" + " cpus/memory along with profile";
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
+ ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
+ + " for component %s";
+ String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
+ "Resource profile is not " + "supported yet. Please specify cpus/memory.";
+
+ String ERROR_NULL_ARTIFACT_ID =
+ "Artifact Id can not be null if artifact type is none";
+ String ERROR_ABSENT_NUM_OF_INSTANCE =
+ "Num of instances should appear either globally or per component";
+ String ERROR_ABSENT_LAUNCH_COMMAND =
+ "Launch_command is required when type is not DOCKER";
+
+ String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
+ + " component level, needs corresponding values set at service level";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
new file mode 100644
index 0000000..e83ccbe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * A service launch exception that includes an exit code;
+ * when caught by the ServiceLauncher, it will convert that
+ * into a process exit code.
+ */
+public class ServiceLaunchException extends YarnException
+ implements ExitCodeProvider, LauncherExitCodes {
+
+ private final int exitCode;
+
+ /**
+ * Create an exception with the specific exit code
+ * @param exitCode exit code
+ * @param cause cause of the exception
+ */
+ public ServiceLaunchException(int exitCode, Throwable cause) {
+ super(cause);
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Create an exception with the specific exit code and text
+ * @param exitCode exit code
+ * @param message message to use in exception
+ */
+ public ServiceLaunchException(int exitCode, String message) {
+ super(message);
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Create an exception with the specific exit code, text and cause
+ * @param exitCode exit code
+ * @param message message to use in exception
+ * @param cause cause of the exception
+ */
+ public ServiceLaunchException(int exitCode, String message, Throwable cause) {
+ super(message, cause);
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Get the exit code
+ * @return the exit code
+ */
+ @Override
+ public int getExitCode() {
+ return exitCode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
new file mode 100644
index 0000000..5b74b80
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+
+public class SliderException extends ServiceLaunchException implements
+ SliderExitCodes {
+ public SliderException() {
+ super(EXIT_EXCEPTION_THROWN, "SliderException");
+ }
+
+ public SliderException(int code, String message) {
+ super(code, message);
+ }
+
+ public SliderException(String s) {
+ super(EXIT_EXCEPTION_THROWN, s);
+ }
+
+ public SliderException(String s, Throwable throwable) {
+ super(EXIT_EXCEPTION_THROWN, s, throwable);
+ }
+
+ /**
+ * Format the exception as you create it
+ * @param code exit code
+ * @param message exception message -sprintf formatted
+ * @param args arguments for the formatting
+ */
+ public SliderException(int code, String message, Object... args) {
+ super(code, String.format(message, args));
+ }
+
+ /**
+ * Format the exception, include a throwable.
+ * The throwable comes before the message so that it is out of the varargs
+ * @param code exit code
+ * @param throwable thrown
+ * @param message message
+ * @param args arguments
+ */
+ public SliderException(int code,
+ Throwable throwable,
+ String message,
+ Object... args) {
+ super(code, String.format(message, args), throwable);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
new file mode 100644
index 0000000..3a9fa25
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.exceptions;
+
+/**
+ * Used to raise a usage exception ... this has the exit code
+ * {@link #EXIT_USAGE}
+ */
+public class UsageException extends SliderException {
+ public UsageException(String s, Object... args) {
+ super(EXIT_USAGE, s, args);
+ }
+
+ public UsageException(Throwable throwable, String message,
+ Object... args) {
+ super(EXIT_USAGE, throwable, message, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
new file mode 100644
index 0000000..33e33a6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.client;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.service.impl.pb.service.ClientAMProtocolPB;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+
+public class ClientAMProtocolPBClientImpl
+ implements ClientAMProtocol, Closeable {
+
+ private ClientAMProtocolPB proxy;
+
+ public ClientAMProtocolPBClientImpl(long clientVersion,
+ InetSocketAddress addr, Configuration conf) throws IOException {
+ RPC.setProtocolEngine(conf, ClientAMProtocolPB.class,
+ ProtobufRpcEngine.class);
+ proxy = RPC.getProxy(ClientAMProtocolPB.class, clientVersion, addr, conf);
+
+ }
+
+ @Override public FlexComponentsResponseProto flexComponents(
+ FlexComponentsRequestProto request) throws IOException, YarnException {
+ try {
+ return proxy.flexComponents(null, request);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public GetStatusResponseProto getStatus(GetStatusRequestProto request)
+ throws IOException, YarnException {
+ try {
+ return proxy.getStatus(null, request);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public StopResponseProto stop(StopRequestProto requestProto)
+ throws IOException, YarnException {
+ try {
+ return proxy.stop(null, requestProto);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override public void close() {
+ if (this.proxy != null) {
+ RPC.stopProxy(this.proxy);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
new file mode 100644
index 0000000..6a9cd37
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.service;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol;
+
+@ProtocolInfo(
+ protocolName = "org.apache.hadoop.yarn.service.ClientAMProtocol",
+ protocolVersion = 1)
+public interface ClientAMProtocolPB extends
+ ClientAMProtocol.ClientAMProtocolService.BlockingInterface {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
new file mode 100644
index 0000000..7100781
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.service;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+
+import java.io.IOException;
+
+public class ClientAMProtocolPBServiceImpl implements ClientAMProtocolPB {
+
+ private ClientAMProtocol real;
+
+ public ClientAMProtocolPBServiceImpl(ClientAMProtocol impl) {
+ this.real = impl;
+ }
+
+ @Override
+ public FlexComponentsResponseProto flexComponents(RpcController controller,
+ FlexComponentsRequestProto request) throws ServiceException {
+ try {
+ return real.flexComponents(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override public GetStatusResponseProto getStatus(RpcController controller,
+ GetStatusRequestProto request) throws ServiceException {
+ try {
+ return real.getStatus(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto stop(
+ RpcController controller,
+ org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto request)
+ throws ServiceException {
+ try {
+ return real.stop(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/ServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/ServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/ServiceMonitor.java
new file mode 100644
index 0000000..982448a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/ServiceMonitor.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.component.ComponentState;
+import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState.STARTED;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.BECOME_NOT_READY;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.BECOME_READY;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState.READY;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_WINDOW;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEFAULT_READINESS_CHECK_INTERVAL;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.READINESS_CHECK_INTERVAL;
+
+public class ServiceMonitor extends AbstractService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceMonitor.class);
+
+ public ScheduledExecutorService executorService;
+ private Map<ContainerId, ComponentInstance> liveInstances = null;
+ private ServiceContext context;
+ private Configuration conf;
+
+ public ServiceMonitor(String name, ServiceContext context) {
+ super(name);
+ liveInstances = context.scheduler.getLiveInstances();
+ this.context = context;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ executorService = Executors.newScheduledThreadPool(1);
+ this.conf = conf;
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ long readinessCheckInterval = YarnServiceConf
+ .getLong(READINESS_CHECK_INTERVAL, DEFAULT_READINESS_CHECK_INTERVAL,
+ context.service.getConfiguration(), conf);
+
+ executorService
+ .scheduleAtFixedRate(new ReadinessChecker(), readinessCheckInterval,
+ readinessCheckInterval, TimeUnit.SECONDS);
+
+ // Default 6 hours.
+ long failureResetInterval = YarnServiceConf
+ .getLong(CONTAINER_FAILURE_WINDOW, 21600,
+ context.service.getConfiguration(), conf);
+
+ executorService
+ .scheduleAtFixedRate(new ContainerFailureReset(), failureResetInterval,
+ failureResetInterval, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void serviceStop() throws Exception {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+ }
+
+ private class ReadinessChecker implements Runnable {
+
+ @Override
+ public void run() {
+
+ // check if the comp instance are ready
+ for (Map.Entry<ContainerId, ComponentInstance> entry : liveInstances
+ .entrySet()) {
+ ComponentInstance instance = entry.getValue();
+
+ ProbeStatus status = instance.ping();
+ if (status.isSuccess()) {
+ if (instance.getState() == STARTED) {
+ // synchronously update the state.
+ instance.handle(
+ new ComponentInstanceEvent(entry.getKey(), BECOME_READY));
+ }
+ } else {
+ if (instance.getState() == READY) {
+ instance.handle(
+ new ComponentInstanceEvent(entry.getKey(), BECOME_NOT_READY));
+ }
+ }
+ }
+
+ for (Component component : context.scheduler.getAllComponents()
+ .values()) {
+ // If comp hasn't started yet and its dependencies are satisfied
+ if (component.getState() == ComponentState.INIT && component
+ .areDependenciesReady()) {
+ LOG.info("[COMPONENT {}]: Dependencies satisfied, ramping up.",
+ component.getName());
+ ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
+ .setDesired(component.getComponentSpec().getNumberOfContainers());
+ component.handle(event);
+ }
+ }
+ }
+ }
+
+ private class ContainerFailureReset implements Runnable {
+ @Override
+ public void run() {
+ for (Component component : context.scheduler.getAllComponents().values()) {
+ component.resetCompFailureCount();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
new file mode 100644
index 0000000..1923086
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Map;
+
+public class HttpProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(HttpProbe.class);
+
+ private static final String HOST_TOKEN = "${THIS_HOST}";
+
+ private final String urlString;
+ private final int timeout;
+ private final int min, max;
+
+
+ public HttpProbe(String url, int timeout, int min, int max, Configuration
+ conf) {
+ super("Http probe of " + url + " [" + min + "-" + max + "]", conf);
+ this.urlString = url;
+ this.timeout = timeout;
+ this.min = min;
+ this.max = max;
+ }
+
+ public static HttpProbe create(Map<String, String> props)
+ throws IOException {
+ String urlString = getProperty(props, WEB_PROBE_URL, null);
+ new URL(urlString);
+ int timeout = getPropertyInt(props, WEB_PROBE_CONNECT_TIMEOUT,
+ WEB_PROBE_CONNECT_TIMEOUT_DEFAULT);
+ int minSuccess = getPropertyInt(props, WEB_PROBE_MIN_SUCCESS,
+ WEB_PROBE_MIN_SUCCESS_DEFAULT);
+ int maxSuccess = getPropertyInt(props, WEB_PROBE_MAX_SUCCESS,
+ WEB_PROBE_MAX_SUCCESS_DEFAULT);
+ return new HttpProbe(urlString, timeout, minSuccess, maxSuccess, null);
+ }
+
+
+ private static HttpURLConnection getConnection(URL url, int timeout) throws
+ IOException {
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setInstanceFollowRedirects(true);
+ connection.setConnectTimeout(timeout);
+ return connection;
+ }
+
+ @Override
+ public ProbeStatus ping(ComponentInstance instance) {
+ ProbeStatus status = new ProbeStatus();
+ ContainerStatus containerStatus = instance.getContainerStatus();
+ if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
+ || StringUtils.isEmpty(containerStatus.getHost())) {
+ status.fail(this, new IOException("IP is not available yet"));
+ return status;
+ }
+
+ String ip = containerStatus.getIPs().get(0);
+ HttpURLConnection connection = null;
+ try {
+ URL url = new URL(urlString.replace(HOST_TOKEN, ip));
+ connection = getConnection(url, this.timeout);
+ int rc = connection.getResponseCode();
+ if (rc < min || rc > max) {
+ String error = "Probe " + url + " error code: " + rc;
+ log.info(error);
+ status.fail(this,
+ new IOException(error));
+ } else {
+ status.succeed(this);
+ }
+ } catch (Throwable e) {
+ String error = "Probe " + urlString + " failed for IP " + ip + ": " + e;
+ log.info(error, e);
+ status.fail(this,
+ new IOException(error, e));
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/LogEntryBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/LogEntryBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/LogEntryBuilder.java
new file mode 100644
index 0000000..9ad86fe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/LogEntryBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+/**
+ * Build up log entries for ease of splunk
+ */
+public class LogEntryBuilder {
+
+ private final StringBuilder builder = new StringBuilder();
+
+ public LogEntryBuilder() {
+ }
+
+ public LogEntryBuilder(String text) {
+ elt(text);
+ }
+
+
+ public LogEntryBuilder(String name, Object value) {
+ entry(name, value);
+ }
+
+ public LogEntryBuilder elt(String text) {
+ addComma();
+ builder.append(text);
+ return this;
+ }
+
+ public LogEntryBuilder elt(String name, Object value) {
+ addComma();
+ entry(name, value);
+ return this;
+ }
+
+ private void addComma() {
+ if (!isEmpty()) {
+ builder.append(", ");
+ }
+ }
+
+ private void entry(String name, Object value) {
+ builder.append(name).append('=');
+ if (value != null) {
+ builder.append('"').append(value.toString()).append('"');
+ } else {
+ builder.append("null");
+ }
+ }
+
+ @Override
+ public String toString() {
+ return builder.toString();
+ }
+
+ private boolean isEmpty() {
+ return builder.length() == 0;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorKeys.java
new file mode 100644
index 0000000..55b55f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorKeys.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+/**
+ * Config keys for monitoring
+ */
+public interface MonitorKeys {
+
+ /**
+ * Port probing key : port to attempt to create a TCP connection to {@value}.
+ */
+ String PORT_PROBE_PORT = "port";
+ /**
+ * Port probing key : timeout for the the connection attempt {@value}.
+ */
+ String PORT_PROBE_CONNECT_TIMEOUT = "timeout";
+ /**
+ * Port probing default : timeout for the connection attempt {@value}.
+ */
+ int PORT_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
+
+ /**
+ * Web probing key : URL {@value}.
+ */
+ String WEB_PROBE_URL = "url";
+ /**
+ * Web probing key : min success code {@value}.
+ */
+ String WEB_PROBE_MIN_SUCCESS = "min.success";
+ /**
+ * Web probing key : max success code {@value}.
+ */
+ String WEB_PROBE_MAX_SUCCESS = "max.success";
+ /**
+ * Web probing default : min successful response code {@value}.
+ */
+ int WEB_PROBE_MIN_SUCCESS_DEFAULT = 200;
+ /**
+ * Web probing default : max successful response code {@value}.
+ */
+ int WEB_PROBE_MAX_SUCCESS_DEFAULT = 299;
+ /**
+ * Web probing key : timeout for the connection attempt {@value}
+ */
+ String WEB_PROBE_CONNECT_TIMEOUT = "timeout";
+ /**
+ * Port probing default : timeout for the connection attempt {@value}.
+ */
+ int WEB_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/58] [abbrv] hadoop git commit: YARN-7126. Create introductory
site documentation for YARN native services. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-7126. Create introductory site documentation for YARN native services. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e660e52a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e660e52a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e660e52a
Branch: refs/heads/yarn-native-services
Commit: e660e52a048eae39f6b57f3c085ebbd34c04dd17
Parents: 7dd4490
Author: Jian He <ji...@apache.org>
Authored: Fri Sep 1 16:19:31 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:14 2017 -0800
----------------------------------------------------------------------
LICENSE.txt | 1 +
.../native-services/NativeServicesIntro.md | 96 +++++++++++++++++++-
2 files changed, 96 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e660e52a/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 67472b6..b0cef03 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1777,6 +1777,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The binary distribution of this product bundles these dependencies under the
following license:
FindBugs-jsr305 3.0.0
+dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved.
--------------------------------------------------------------------------------
(2-clause BSD)
Redistribution and use in source and binary forms, with or without
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e660e52a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
index 89fefe9..e6a4e91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
@@ -10,4 +10,98 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
--->
\ No newline at end of file
+-->
+
+# Introduction: YARN Native Services
+
+## Overview
+YARN Native Services provides first class framework support and APIs to host long running services natively in YARN. In addition to launching services, the new APIs support performing lifecycle management operations, such as flex service components up/down, manage lifetime, upgrade the service to a newer version, and stop/restart/delete the service.
+
+The native services capabilities are built on the existing low-level resource management API provided by YARN that can support any type of application. Other application frameworks like Hadoop MapReduce already expose higher level APIs that users can leverage to run applications on top of YARN. With the advent of containerization technologies like Docker, providing first class support and APIs for long running services at the framework level made sense.
+
+Relying on a framework has the advantage of exposing a simpler usage model to the user by enabling service configuration and launch through specification (without writing new code), as well as hiding complex low-level details including state management and fault-tolerance etc. Users/operators of existing services typically like to avoid modifying an existing service to be aware of YARN. With first class support capable of running a single Docker image as well as complex assemblies comprised of multiple Docker images, there is no need for service owners to be aware of YARN. Developers of new services do not have to worry about YARN internals and only need to focus on containerization of their service(s).
+
+## First class support for services
+In order to natively provide first class support for long running services, several new features and improvements have been made at the framework level.
+
+### Incorporate Apache Slider into Apache YARN
+Apache Slider, which existed as a separate incubator project has been merged into YARN to kick start the first class support. Apache Slider is a universal Application Master (AM) which had several key features built in - fault tolerance of service containers and AM, work-preserving AM restarts, service logs management, service management like flex up/down, stop/start, and rolling upgrade to newer service versions, etc. Of course lot more work has been done on top of what Apache Slider brought in, details of which follow.
+
+### Native Services API
+A significant effort has gone into simplifying the user facing story for building services. In the past, bringing a new service to YARN was not a pleasant experience. The APIs of existing frameworks are either too low-level (native YARN), require writing new code (for frameworks with programmatic APIs) or require writing a complex spec (for declarative frameworks).
+
+The new REST APIs are very simple to use. The REST layer acts as a single point of entry for creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex, multi-component applications needing special orchestration needs.
+
+Plan is to make this a unified REST based entry point for other important features like resource-profile management ([YARN-3926](https://issues.apache.org/jira/browse/YARN-4793)), package-definitions' lifecycle-management and service-discovery ([YARN-913](https://issues.apache.org/jira/browse/YARN-913)/[YARN-4757](https://issues.apache.org/jira/browse/YARN-4757)).
+
+### Native Services Discovery
+The new discovery solution exposes the registry information through a more generic and widely used mechanism: DNS. Service Discovery via DNS uses the well-known DNS interfaces to browse the network for services. Having the registry information exposed via DNS simplifies the life of services.
+
+The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface. In practice, this made it very difficult for wiring up existing clients and services. For e.g., dynamic configuration of dependent endpoints of a service was not easy to implement using the registry-read mechanisms, **without** code-changes to existing services. These are solved by the DNS based service discovery.
+
+### Scheduling
+[YARN-6592](https://issues.apache.org/jira/browse/YARN-6592) covers a host of scheduling features that are useful for short-running applications and services alike. Below, are a few very important YARN core features that help schedule services better. Without these, running services on YARN is a hassle.
+
+* Affinity (TBD)
+* Anti-affinity (TBD)
+* Gang scheduling (TBD)
+* Malleable container sizes ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197))
+
+### Resource Profiles
+YARN always had support for memory as a resource, inheriting it from Hadoop-(1.x)’s MapReduce platform. Later support for CPU as a resource ([YARN-2](https://issues.apache.org/jira/browse/YARN-2)/[YARN-3](https://issues.apache.org/jira/browse/YARN-3)) was added. Multiple efforts added support for various other resource-types in YARN such as disk ([YARN-2139](https://issues.apache.org/jira/browse/YARN-2139)), and network ([YARN-2140](https://issues.apache.org/jira/browse/YARN-2140)), specifically benefiting long running services.
+
+In many systems outside of YARN, users are already accustomed to specifying their desired ‘box’ of requirements where each box comes with a predefined amount of each resources. Admins would define various available box-sizes (small, medium, large etc) and users would pick the ones they desire and everybody is happy. In [YARN-3926](https://issues.apache.org/jira/browse/YARN-3926), YARN introduces Resource Profiles which extends the YARN resource model for easier resource-type management and profiles. This helps in two ways - the system can schedule applications better and it can perform intelligent over-subscription of resources where applicable.
+
+Resource profiles are all the more important for services since -
+* Similar to short running apps, you don’t have to fiddle with varying resource-requirements for each container type
+* Services usually end up planning for peak usages, leaving a lot of possibility of barren utilization
+
+### Special handling of preemption and container reservations
+TBD.
+
+Preemption and reservation of long running containers have different implications from regular ones. Preemption of resources in YARN today works by killing of containers. For long-lived services this is unacceptable. Also, scheduler should avoid allocating long running containers on borrowed resources. [YARN-4724](https://issues.apache.org/jira/browse/YARN-4724) will address some of these special recognition of service containers.
+
+### Container auto-restarts
+If a service container dies, expiring container's allocation and releasing the allocation is undesirable in many cases. Long running containers may exit for various reasons, crash and need to restart but forcing them to go through the complete scheduling cycle, resource localization, etc. is both unnecessary and expensive.
+
+Services can enable app-specific policies to prevent NodeManagers to automatically restart containers. [YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails.
+
+### Container allocation re-use for application upgrades
+TBD.
+
+Auto-restart of containers will support upgrade of service containers without reclaiming the resources first. During an upgrade, with multitude of other applications running in the system, giving up and getting back resources allocated to the service is hard to manage. Node-Labels help this cause but are not straight-forward to use to address the app-specific use-cases. The umbrella [YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) along with [YARN-5620](https://issues.apache.org/jira/browse/YARN-5620) and [YARN-4470](https://issues.apache.org/jira/browse/YARN-4470) will take care of this.
+
+### Dynamic Configurations
+Most production-level services require dynamic configurations to manage and simplify their lifecycle. Container’s resource size, local/work dirs and log-dirs are the most basic information services need. Service's endpoint details (host/port), their inter-component dependencies, health-check endpoints, etc. are all critical to the success of today's real-life services.
+
+### Resource re-localization for reconfiguration/upgrades
+TBD
+
+### Service Registry
+TBD
+
+### Service persistent storage and volume support
+TBD
+
+### Packaging
+TBD
+
+### Container image registry (private, public and hybrid)
+TBD
+
+### Container image management and APIs
+TBD
+
+### Container image storage
+TBD
+
+### Monitoring
+TBD
+
+### Metrics
+TBD
+
+### Service Logs
+TBD
+
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/58] [abbrv] hadoop git commit: YARN-7186. Add examples in
yarn-service. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7186. Add examples in yarn-service. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37c9b732
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37c9b732
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37c9b732
Branch: refs/heads/yarn-native-services
Commit: 37c9b7327d188ccad7fd36b7466a65f68ad0c899
Parents: 307d55b
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Sep 14 15:43:00 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:16 2017 -0800
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 7 ++++
.../examples/sleeper/sleeper.json | 15 +++++++
.../yarn/service/client/ServiceClient.java | 26 ++++++++++--
.../client/params/AbstractActionArgs.java | 2 +-
.../AbstractClusterBuildingActionArgs.java | 2 +-
.../service/client/params/ActionBuildArgs.java | 7 +++-
.../service/client/params/ActionCreateArgs.java | 12 +++++-
.../service/client/params/ActionExamples.java | 26 ------------
.../yarn/service/client/params/Arguments.java | 4 +-
.../yarn/service/client/params/ClientArgs.java | 44 --------------------
.../yarn/service/client/params/CommonArgs.java | 15 +------
.../service/client/params/SliderActions.java | 2 -
.../src/site/markdown/YarnCommands.md | 34 ++++++++-------
.../site/markdown/yarn-service/QuickStart.md | 9 +++-
14 files changed, 91 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 8b3d292..83633ac 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -98,6 +98,13 @@
<outputDirectory>etc/hadoop</outputDirectory>
</fileSet>
<fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/yarn-service-examples</outputDirectory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ </fileSet>
+ <fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
new file mode 100644
index 0000000..89ce527
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
@@ -0,0 +1,15 @@
+{
+ "name": "sleeper-service",
+ "components" :
+ [
+ {
+ "name": "sleeper",
+ "number_of_containers": 2,
+ "launch_command": "sleep 900000",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index bfddc44..6890aef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
import org.apache.hadoop.yarn.service.client.params.Arguments;
@@ -160,12 +161,15 @@ public class ServiceClient extends CompositeService
AbstractClusterBuildingActionArgs args) throws IOException {
File file = args.getFile();
Path filePath = new Path(file.getAbsolutePath());
- LOG.info("Loading app json from: " + filePath);
+ LOG.info("Loading service definition from: " + filePath);
Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
if (args.lifetime > 0) {
service.setLifetime(args.lifetime);
}
+ if (!StringUtils.isEmpty(args.getServiceName())) {
+ service.setName(args.getServiceName());
+ }
return service;
}
@@ -182,9 +186,23 @@ public class ServiceClient extends CompositeService
return EXIT_SUCCESS;
}
- public int actionCreate(AbstractClusterBuildingActionArgs args)
+ public int actionCreate(ActionCreateArgs args)
throws IOException, YarnException {
- actionCreate(loadAppJsonFromLocalFS(args));
+ Service serviceDef;
+ if (args.file != null) {
+ serviceDef = loadAppJsonFromLocalFS(args);
+ } else if (!StringUtils.isEmpty(args.example)) {
+ // create an example service
+ String yarnHome = System
+ .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
+ args.file = new File(MessageFormat
+ .format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
+ yarnHome, args.example, args.example));
+ serviceDef = loadAppJsonFromLocalFS(args);
+ } else {
+ throw new YarnException("No service definition provided!");
+ }
+ actionCreate(serviceDef);
return EXIT_SUCCESS;
}
@@ -213,7 +231,7 @@ public class ServiceClient extends CompositeService
Map<String, Long> componentCounts =
new HashMap<>(flexArgs.getComponentMap().size());
Service persistedService =
- ServiceApiUtil.loadService(fs, flexArgs.getClusterName());
+ ServiceApiUtil.loadService(fs, flexArgs.getServiceName());
if (!StringUtils.isEmpty(persistedService.getId())) {
cachedAppIds.put(persistedService.getName(),
ApplicationId.fromString(persistedService.getId()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
index 8b2d93e..9b7e2a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
@@ -50,7 +50,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
* get the name: relies on arg 1 being the cluster name in all operations
* @return the name argument, null if there is none
*/
- public String getClusterName() {
+ public String getServiceName() {
return (parameters.isEmpty()) ? null : parameters.get(0);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
index d215436..434c6d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -28,7 +28,7 @@ import java.io.File;
*/
public abstract class AbstractClusterBuildingActionArgs
extends AbstractActionArgs {
- @Parameter(names = { ARG_FILE, ARG_FILE_SHORT }, required = true,
+ @Parameter(names = { ARG_FILE, ARG_FILE_SHORT },
description = "The path to the service definition file in JSON format.")
public File file;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
index cf61525..0228125 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
@Parameters(commandNames = { SliderActions.ACTION_BUILD},
commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
@@ -30,7 +31,9 @@ public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
}
@Override
- public int getMinParams() {
- return 0;
+ public void validate() throws BadCommandArgumentsException {
+ if (file == null) {
+ throw new BadCommandArgumentsException("No service definition provided.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
index c43f4fc..039f528 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
@@ -18,21 +18,29 @@
package org.apache.hadoop.yarn.service.client.params;
+import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
@Parameters(commandNames = { SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
+ @Parameter(names = { ARG_EXAMPLE, ARG_EXAMPLE_SHORT },
+ description = "The name of the example service such as sleeper")
+ public String example;
+
@Override
public String getActionName() {
return SliderActions.ACTION_CREATE;
}
@Override
- public int getMinParams() {
- return 0;
+ public void validate() throws BadCommandArgumentsException {
+ if (file == null && example == null) {
+ throw new BadCommandArgumentsException("No service definition provided.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
deleted file mode 100644
index e489e17..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_EXAMPLES},
- commandDescription = SliderActions.DESCRIBE_ACTION_EXAMPLES)
-public class ActionExamples {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
index 911440d..9672621 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -42,8 +42,8 @@ public interface Arguments {
String ARG_DELETE = "--delete";
String ARG_DEST = "--dest";
String ARG_DESTDIR = "--destdir";
- String ARG_FILESYSTEM = "--fs";
- String ARG_FILESYSTEM_LONG = "--filesystem";
+ String ARG_EXAMPLE = "--example";
+ String ARG_EXAMPLE_SHORT = "-e";
String ARG_FOLDER = "--folder";
String ARG_FORCE = "--force";
String ARG_FORMAT = "--format";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
index f479cd2..84d3bbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@@ -104,51 +102,10 @@ public class ClientArgs extends CommonArgs {
return actionDependencyArgs;
}
- public ActionDestroyArgs getActionDestroyArgs() {
- return actionDestroyArgs;
- }
-
- public ActionExistsArgs getActionExistsArgs() {
- return actionExistsArgs;
- }
-
public ActionFlexArgs getActionFlexArgs() {
return actionFlexArgs;
}
- public ActionFreezeArgs getActionFreezeArgs() {
- return actionFreezeArgs;
- }
-
- public ActionListArgs getActionListArgs() {
- return actionListArgs;
- }
-
-
- public ActionRegistryArgs getActionRegistryArgs() {
- return actionRegistryArgs;
- }
-
- public ActionResolveArgs getActionResolveArgs() {
- return actionResolveArgs;
- }
-
- public ActionResourceArgs getActionResourceArgs() {
- return actionResourceArgs;
- }
-
- public ActionStatusArgs getActionStatusArgs() {
- return actionStatusArgs;
- }
-
- public ActionThawArgs getActionThawArgs() {
- return actionThawArgs;
- }
-
- public ActionTokensArgs getActionTokenArgs() {
- return actionTokenArgs;
- }
-
/**
* Look at the chosen action and bind it as the core action for the operation.
* @throws SliderException bad argument or similar
@@ -227,7 +184,6 @@ public class ClientArgs extends CommonArgs {
case ACTION_UPDATE:
bindCoreAction(actionUpdateArgs);
break;
-
default:
throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ " " + action);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
index a5638a4..145b44a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
@@ -24,7 +24,6 @@ import com.beust.jcommander.ParameterDescription;
import com.beust.jcommander.ParameterException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@@ -88,7 +87,7 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
* @return the name argument, null if there is none
*/
public String getClusterName() {
- return coreAction.getClusterName();
+ return coreAction.getServiceName();
}
protected CommonArgs(String[] args) {
@@ -142,10 +141,6 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
return result;
}
- public static String usage(CommonArgs serviceArgs) {
- return usage(serviceArgs, null);
- }
-
/**
* Parse routine -includes registering the action-specific argument classes
* and postprocess it
@@ -164,14 +159,6 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
postProcess();
}
- /**
- * Add a command
- * @param name action
- * @param arg value
- */
- protected void addAction(String name, Object arg) {
- commander.addCommand(name, arg);
- }
protected void addActions(Object... actions) {
for (Object action : actions) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
index fa05f2e..680bc0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
@@ -30,7 +30,6 @@ public interface SliderActions {
String ACTION_UPGRADE = "upgrade";
String ACTION_DESTROY = "destroy";
String ACTION_EXISTS = "exists";
- String ACTION_EXAMPLES = "examples";
String ACTION_FLEX = "flex";
String ACTION_STOP = "stop";
String ACTION_HELP = "help";
@@ -59,7 +58,6 @@ public interface SliderActions {
"Destroy a stopped service, service must be stopped first before destroying.";
String DESCRIBE_ACTION_EXISTS =
"Probe for a service running";
- String DESCRIBE_ACTION_EXAMPLES = "Run an example service on YARN";
String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
String DESCRIBE_ACTION_FREEZE =
"Stop a running service";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index ca46c8c..caeaeb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -76,28 +76,32 @@ Usage `yarn service [sub-command] [service-name] [options]`
* `build`: Build a service with its specifications, but do not start it.
```
- Usage: yarn service build --file [file]
- ```
+ Usage: yarn service build [service-name] --file [file]
+ Fields:
+ service-name Optional. If specified, it will override the name in the service definition.
- | COMMAND\_OPTIONS | Description |
- |:---- |:---- |
- | --file or -f | The local path to the service definition file |
+ Options:
+ --file,-f The local path to the service definition file
+ ```
* `create`: create a service, it's equivalent to first invoke build and then start.
```
- Usage: yarn service create --file [file]
+ Usage: yarn service create [service-name] --file [file]
+ Fields:
+ service-name Optional. If specified, it will override the name in the service definition.
+
+ Options:
+ --file,-f The local path to the service definition file.
+ --example,-e The name of the example service such as:
+ Sleeper A simple service that launches a few non-docker sleep containers on YARN.
```
- | COMMAND\_OPTIONS | Description |
- |:---- |:---- |
- | --file or -f | The local path to the service definition file |
* `dependency`: Yarn service framework dependency (libraries) management.
```
Usage: yarn service dependency [options]
+ Option:
+ --upload Pre-upload the dependency jars onto HDFS to expediate service launch process.
```
- | COMMAND\_OPTIONS | Description |
- |:---- |:---- |
- | --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. |
* `destroy`: Destroy a stopped service, service must be stopped first before destroying.
```
@@ -106,10 +110,10 @@ Usage `yarn service [sub-command] [service-name] [options]`
* `flex`: Flex a service's component by increasing or decreasing the number of containers.
```
Usage: yarn service flex [service-name] --component [component-name] [count]
+ Options:
+ --component [component-name] [count]
+ Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.
```
- | COMMAND\_OPTIONS | Description |
- |:---- |:---- |
- | --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.|
* `status`: Get the status of a service.
```
Usage: yarn service status [service-name]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index 327566b..ab415de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -43,7 +43,14 @@ Below is a simple service definition that launches sleep containers on YARN by w
]
}
```
-
+User can simply run a pre-built example service on YARN using below command:
+```
+yarn service create [service-name] --example [example-name]
+```
+e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN.
+```
+yarn service create my-sleeper --example sleeper
+```
For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
## Manage services on YARN via CLI
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[54/58] [abbrv] hadoop git commit: YARN-7371. Added allocateRequestId
in NMContainerStatus for recovery. Contributed by Chandni Singh
Posted by ji...@apache.org.
YARN-7371. Added allocateRequestId in NMContainerStatus for recovery. Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a55d0738
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a55d0738
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a55d0738
Branch: refs/heads/yarn-native-services
Commit: a55d0738f1eba225c45578bbdabd0e5a6d6f5392
Parents: a127f7b
Author: Jian He <ji...@apache.org>
Authored: Sat Nov 4 23:15:21 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:20 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceTestUtils.java | 10 +-
.../yarn/service/TestYarnNativeServices.java | 105 ++++++++++++++++---
.../yarn/security/ContainerTokenIdentifier.java | 20 +++-
.../src/main/proto/yarn_security_token.proto | 1 +
.../api/protocolrecords/NMContainerStatus.java | 18 +++-
.../impl/pb/NMContainerStatusPBImpl.java | 15 ++-
.../OpportunisticContainerAllocator.java | 2 +-
.../yarn_server_common_service_protos.proto | 1 +
.../containermanager/ContainerManagerImpl.java | 5 +-
.../container/ContainerImpl.java | 8 +-
.../scheduler/AbstractYarnScheduler.java | 1 +
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../security/RMContainerTokenSecretManager.java | 36 ++-----
.../server/resourcemanager/TestRMRestart.java | 2 +-
.../TestResourceTrackerService.java | 6 +-
.../capacity/TestContainerAllocation.java | 5 +-
16 files changed, 177 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 19a5177..cf32880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -131,12 +131,16 @@ public class ServiceTestUtils {
return fs;
}
+ protected MiniYARNCluster getYarnCluster() {
+ return yarnCluster;
+ }
+
protected void setupInternal(int numNodeManager)
throws Exception {
LOG.info("Starting up YARN cluster");
-// Logger rootLogger = LogManager.getRootLogger();
-// rootLogger.setLevel(Level.DEBUG);
- setConf(new YarnConfiguration());
+ if (conf == null) {
+ setConf(new YarnConfiguration());
+ }
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
// reduce the teardown waiting time
conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 64fcf57..f98d90a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -18,12 +18,13 @@
package org.apache.hadoop.yarn.service;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Component;
@@ -41,17 +42,12 @@ import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
+import java.util.*;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
/**
* End to end tests to test deploying services with MiniYarnCluster and a in-JVM
@@ -67,7 +63,8 @@ public class TestYarnNativeServices extends ServiceTestUtils {
@Before
public void setup() throws Exception {
- setupInternal(NUM_NMS);
+ File tmpYarnDir = new File("target", "tmp");
+ FileUtils.deleteQuietly(tmpYarnDir);
}
@After
@@ -84,6 +81,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
// 6. Destroy the service
@Test (timeout = 200000)
public void testCreateFlexStopDestroyService() throws Exception {
+ setupInternal(NUM_NMS);
ServiceClient client = createClient();
Service exampleApp = createExampleApplication();
client.actionCreate(exampleApp);
@@ -135,6 +133,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
// Check containers for compa started before containers for compb
@Test (timeout = 200000)
public void testComponentStartOrder() throws Exception {
+ setupInternal(NUM_NMS);
ServiceClient client = createClient();
Service exampleApp = new Service();
exampleApp.setName("teststartorder");
@@ -155,6 +154,70 @@ public class TestYarnNativeServices extends ServiceTestUtils {
client.actionDestroy(exampleApp.getName());
}
+ // Test to verify recovery of SeviceMaster after RM is restarted.
+ // 1. Create an example service.
+ // 2. Restart RM.
+ // 3. Fail the application attempt.
+ // 4. Verify ServiceMaster recovers.
+ @Test(timeout = 200000)
+ public void testRecoverComponentsAfterRMRestart() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+ conf.setBoolean(
+ YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
+ conf.setLong(YarnConfiguration.NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ 500L);
+
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
+ setConf(conf);
+ setupInternal(NUM_NMS);
+
+ ServiceClient client = createClient();
+ Service exampleApp = createExampleApplication();
+ client.actionCreate(exampleApp);
+ waitForAllCompToBeReady(client, exampleApp);
+
+ LOG.info("Restart the resource manager");
+ getYarnCluster().restartResourceManager(
+ getYarnCluster().getActiveRMIndex());
+ GenericTestUtils.waitFor(() ->
+ getYarnCluster().getResourceManager().getServiceState() ==
+ org.apache.hadoop.service.Service.STATE.STARTED, 2000, 200000);
+ Assert.assertTrue("node managers connected",
+ getYarnCluster().waitForNodeManagersToConnect(5000));
+
+ ApplicationId exampleAppId = ApplicationId.fromString(exampleApp.getId());
+ ApplicationAttemptId applicationAttemptId = client.getYarnClient()
+ .getApplicationReport(exampleAppId).getCurrentApplicationAttemptId();
+
+ Multimap<String, String> containersBeforeFailure = getContainersForAllComp(
+ client, exampleApp);
+
+ LOG.info("Fail the application attempt {}", applicationAttemptId);
+ client.getYarnClient().failApplicationAttempt(applicationAttemptId);
+ //wait until attempt 2 is running
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ApplicationReport ar = client.getYarnClient()
+ .getApplicationReport(exampleAppId);
+ return ar.getCurrentApplicationAttemptId().getAttemptId() == 2 &&
+ ar.getYarnApplicationState() == YarnApplicationState.RUNNING;
+ } catch (YarnException | IOException e) {
+ throw new RuntimeException("while waiting", e);
+ }
+ }, 2000, 200000);
+
+ Multimap<String, String> containersAfterFailure = getContainersForAllComp(
+ client, exampleApp);
+ Assert.assertEquals("component container affected by restart",
+ containersBeforeFailure, containersAfterFailure);
+
+ LOG.info("Stop/destroy service {}", exampleApp);
+ client.actionStop(exampleApp.getName(), true);
+ client.actionDestroy(exampleApp.getName());
+ }
+
// Check containers launched are in dependency order
// Get all containers into a list and sort based on container launch time e.g.
// compa-c1, compa-c2, compb-c1, compb-c2;
@@ -297,6 +360,24 @@ public class TestYarnNativeServices extends ServiceTestUtils {
}, 2000, 200000);
}
+ /**
+ * Get all containers of a service.
+ */
+ private Multimap<String, String> getContainersForAllComp(ServiceClient client,
+ Service example) throws IOException, YarnException {
+
+ Multimap<String, String> allContainers = HashMultimap.create();
+ Service retrievedApp = client.getStatus(example.getName());
+ retrievedApp.getComponents().forEach(component -> {
+ if (component.getContainers() != null) {
+ component.getContainers().forEach(container -> {
+ allContainers.put(component.getName(), container.getId());
+ });
+ }
+ });
+ return allContainers;
+ }
+
private ServiceClient createClient() throws Exception {
ServiceClient client = new ServiceClient() {
@Override protected Path addJarResource(String appName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 07917ef..9e7d132 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -115,7 +115,7 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
this(containerID, 0, hostName, appSubmitter, r, expiryTimeStamp,
masterKeyId, rmIdentifier, priority, creationTime,
logAggregationContext, nodeLabelExpression, containerType,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, -1);
}
public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
@@ -123,6 +123,19 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
int masterKeyId, long rmIdentifier, Priority priority, long creationTime,
LogAggregationContext logAggregationContext, String nodeLabelExpression,
ContainerType containerType, ExecutionType executionType) {
+
+ this(containerID, containerVersion, hostName, appSubmitter, r,
+ expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime,
+ logAggregationContext, nodeLabelExpression, containerType,
+ executionType, -1);
+ }
+
+ public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
+ String hostName, String appSubmitter, Resource r, long expiryTimeStamp,
+ int masterKeyId, long rmIdentifier, Priority priority, long creationTime,
+ LogAggregationContext logAggregationContext, String nodeLabelExpression,
+ ContainerType containerType, ExecutionType executionType,
+ long allocationRequestId) {
ContainerTokenIdentifierProto.Builder builder =
ContainerTokenIdentifierProto.newBuilder();
if (containerID != null) {
@@ -152,6 +165,7 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
}
builder.setContainerType(convertToProtoFormat(containerType));
builder.setExecutionType(convertToProtoFormat(executionType));
+ builder.setAllocationRequestId(allocationRequestId);
proto = builder.build();
}
@@ -243,6 +257,10 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
return new LogAggregationContextPBImpl(proto.getLogAggregationContext());
}
+ public long getAllocationRequestId() {
+ return proto.getAllocationRequestId();
+ }
+
@Override
public void write(DataOutput out) throws IOException {
LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
index 851920d..d8288ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
@@ -53,6 +53,7 @@ message ContainerTokenIdentifierProto {
optional ContainerTypeProto containerType = 12;
optional ExecutionTypeProto executionType = 13 [default = GUARANTEED];
optional int32 version = 14 [default = 0];
+ optional int64 allocation_request_id = 15 [default = -1];
}
message ClientToAMTokenIdentifierProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
index 180add8..1a095f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
@@ -41,14 +41,14 @@ public abstract class NMContainerStatus {
long creationTime) {
return newInstance(containerId, version, containerState, allocatedResource,
diagnostics, containerExitStatus, priority, creationTime,
- CommonNodeLabelsManager.NO_LABEL, ExecutionType.GUARANTEED);
+ CommonNodeLabelsManager.NO_LABEL, ExecutionType.GUARANTEED, -1);
}
public static NMContainerStatus newInstance(ContainerId containerId,
int version, ContainerState containerState, Resource allocatedResource,
String diagnostics, int containerExitStatus, Priority priority,
long creationTime, String nodeLabelExpression,
- ExecutionType executionType) {
+ ExecutionType executionType, long allocationRequestId) {
NMContainerStatus status =
Records.newRecord(NMContainerStatus.class);
status.setContainerId(containerId);
@@ -61,6 +61,7 @@ public abstract class NMContainerStatus {
status.setCreationTime(creationTime);
status.setNodeLabelExpression(nodeLabelExpression);
status.setExecutionType(executionType);
+ status.setAllocationRequestId(allocationRequestId);
return status;
}
@@ -130,6 +131,19 @@ public abstract class NMContainerStatus {
public abstract void setNodeLabelExpression(
String nodeLabelExpression);
+ /**
+ * @return the <em>ID</em> corresponding to the original allocation request.
+ */
+ public abstract long getAllocationRequestId();
+
+ /**
+ * Set the <em>ID</em> corresponding to the original allocation request.
+ *
+ * @param allocationRequestId the <em>ID</em> corresponding to the original
+ * allocation request.
+ */
+ public abstract void setAllocationRequestId(long allocationRequestId);
+
public int getVersion() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
index 38df5f6..8ed02fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
@@ -91,7 +91,8 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
.append("Diagnostics: ").append(getDiagnostics()).append(", ")
.append("ExitStatus: ").append(getContainerExitStatus()).append(", ")
.append("NodeLabelExpression: ").append(getNodeLabelExpression())
- .append("Priority: ").append(getPriority())
+ .append("Priority: ").append(getPriority()).append(", ")
+ .append("AllocationRequestId: ").append(getAllocationRequestId())
.append("]");
return sb.toString();
}
@@ -270,6 +271,18 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
builder.setExecutionType(convertToProtoFormat(executionType));
}
+ @Override
+ public long getAllocationRequestId() {
+ NMContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getAllocationRequestId());
+ }
+
+ @Override
+ public void setAllocationRequestId(long allocationRequestId) {
+ maybeInitBuilder();
+ builder.setAllocationRequestId(allocationRequestId);
+ }
+
private void mergeLocalToBuilder() {
if (this.containerId != null
&& !((ContainerIdPBImpl) containerId).getProto().equals(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
index ede4958..1f53648 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
@@ -564,7 +564,7 @@ public class OpportunisticContainerAllocator {
tokenSecretManager.getCurrentKey().getKeyId(), rmIdentifier,
schedulerKey.getPriority(), currTime,
null, CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK,
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, schedulerKey.getAllocationRequestId());
byte[] pwd =
tokenSecretManager.createPassword(containerTokenIdentifier);
Token containerToken = newContainerToken(node.getNodeId(), pwd,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 8e59f14..8c4fc69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -176,6 +176,7 @@ message NMContainerStatusProto {
optional string nodeLabelExpression = 8;
optional int32 version = 9;
optional ExecutionTypeProto executionType = 10 [default = GUARANTEED];
+ optional int64 allocation_request_id = 11 [default = -1];
}
message SCMUploaderNotifyRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 55119e0..831ba0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -437,7 +437,7 @@ public class ContainerManagerImpl extends CompositeService implements
throws IOException {
StartContainerRequest req = rcs.getStartRequest();
ContainerLaunchContext launchContext = req.getContainerLaunchContext();
- ContainerTokenIdentifier token = null;
+ ContainerTokenIdentifier token;
if(rcs.getCapability() != null) {
ContainerTokenIdentifier originalToken =
BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
@@ -449,7 +449,8 @@ public class ContainerManagerImpl extends CompositeService implements
originalToken.getCreationTime(),
originalToken.getLogAggregationContext(),
originalToken.getNodeLabelExpression(),
- originalToken.getContainerType(), originalToken.getExecutionType());
+ originalToken.getContainerType(), originalToken.getExecutionType(),
+ originalToken.getAllocationRequestId());
} else {
token = BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 17b24b4..901b013 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -844,12 +844,14 @@ public class ContainerImpl implements Container {
public NMContainerStatus getNMContainerStatus() {
this.readLock.lock();
try {
- return NMContainerStatus.newInstance(this.containerId, this.version,
- getCurrentState(), getResource(), diagnostics.toString(), exitCode,
+ return NMContainerStatus.newInstance(this.containerId,
+ this.version, getCurrentState(), getResource(),
+ diagnostics.toString(), exitCode,
containerTokenIdentifier.getPriority(),
containerTokenIdentifier.getCreationTime(),
containerTokenIdentifier.getNodeLabelExpression(),
- containerTokenIdentifier.getExecutionType());
+ containerTokenIdentifier.getExecutionType(),
+ containerTokenIdentifier.getAllocationRequestId());
} finally {
this.readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 8ce6eb8..53f43e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -571,6 +571,7 @@ public abstract class AbstractYarnScheduler
status.getPriority(), null);
container.setVersion(status.getVersion());
container.setExecutionType(status.getExecutionType());
+ container.setAllocationRequestId(status.getAllocationRequestId());
ApplicationAttemptId attemptId =
container.getId().getApplicationAttemptId();
RMContainer rmContainer = new RMContainerImpl(container,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index db63cd8..ce71afa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -655,7 +655,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
container.getNodeId(), getUser(), container.getResource(),
container.getPriority(), rmContainer.getCreationTime(),
this.logAggregationContext, rmContainer.getNodeLabelExpression(),
- containerType, container.getExecutionType()));
+ containerType, container.getExecutionType(),
+ container.getAllocationRequestId()));
updateNMToken(container);
} catch (IllegalArgumentException e) {
// DNS might be down, skip returning this container.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
index 677aa14..191900b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
@@ -167,7 +167,7 @@ public class RMContainerTokenSecretManager extends
}
/**
- * Helper function for creating ContainerTokens
+ * Helper function for creating ContainerTokens.
*
* @param containerId Container Id
* @param containerVersion Container Version
@@ -183,37 +183,13 @@ public class RMContainerTokenSecretManager extends
Resource capability, Priority priority, long createTime) {
return createContainerToken(containerId, containerVersion, nodeId,
appSubmitter, capability, priority, createTime,
- null, null, ContainerType.TASK);
+ null, null, ContainerType.TASK,
+ ExecutionType.GUARANTEED, -1);
}
/**
* Helper function for creating ContainerTokens.
*
- * @param containerId containerId.
- * @param containerVersion containerVersion.
- * @param nodeId nodeId.
- * @param appSubmitter appSubmitter.
- * @param capability capability.
- * @param priority priority.
- * @param createTime createTime.
- * @param logAggregationContext logAggregationContext.
- * @param nodeLabelExpression nodeLabelExpression.
- * @param containerType containerType.
- * @return the container-token.
- */
- public Token createContainerToken(ContainerId containerId,
- int containerVersion, NodeId nodeId, String appSubmitter,
- Resource capability, Priority priority, long createTime,
- LogAggregationContext logAggregationContext, String nodeLabelExpression,
- ContainerType containerType) {
- return createContainerToken(containerId, containerVersion, nodeId,
- appSubmitter, capability, priority, createTime, null, null,
- ContainerType.TASK, ExecutionType.GUARANTEED);
- }
-
- /**
- * Helper function for creating ContainerTokens
- *
* @param containerId Container Id
* @param containerVersion Container version
* @param nodeId Node Id
@@ -225,13 +201,15 @@ public class RMContainerTokenSecretManager extends
* @param nodeLabelExpression Node Label Expression
* @param containerType Container Type
* @param execType Execution Type
+ * @param allocationRequestId allocationRequestId
* @return the container-token
*/
public Token createContainerToken(ContainerId containerId,
int containerVersion, NodeId nodeId, String appSubmitter,
Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext, String nodeLabelExpression,
- ContainerType containerType, ExecutionType execType) {
+ ContainerType containerType, ExecutionType execType,
+ long allocationRequestId) {
byte[] password;
ContainerTokenIdentifier tokenIdentifier;
long expiryTimeStamp =
@@ -246,7 +224,7 @@ public class RMContainerTokenSecretManager extends
this.currentMasterKey.getMasterKey().getKeyId(),
ResourceManager.getClusterTimeStamp(), priority, createTime,
logAggregationContext, nodeLabelExpression, containerType,
- execType);
+ execType, allocationRequestId);
password = this.createPassword(tokenIdentifier);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index f0f51f3..a4f18ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -2113,7 +2113,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
NMContainerStatus.newInstance(containerId, 0, containerState,
Resource.newInstance(1024, 1), "recover container", 0,
Priority.newInstance(0), 0, nodeLabelExpression,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, -1);
return containerReport;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 2bd745b..1cb2d0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -2088,17 +2088,17 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
NMContainerStatus.newInstance(c1, 1, ContainerState.RUNNING,
Resource.newInstance(1024, 1), "Dummy Queued OC",
ContainerExitStatus.INVALID, Priority.newInstance(5), 1234, "",
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, -1);
NMContainerStatus runningOpp =
NMContainerStatus.newInstance(c2, 1, ContainerState.RUNNING,
Resource.newInstance(2048, 1), "Dummy Running OC",
ContainerExitStatus.INVALID, Priority.newInstance(6), 1234, "",
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, -1);
NMContainerStatus runningGuar =
NMContainerStatus.newInstance(c3, 1, ContainerState.RUNNING,
Resource.newInstance(2048, 1), "Dummy Running GC",
ContainerExitStatus.INVALID, Priority.newInstance(6), 1234, "",
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, -1);
req.setContainerStatuses(Arrays.asList(queuedOpp, runningOpp, runningGuar));
// trying to register a invalid node.
RegisterNodeManagerResponse response =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index b1ca72a..6f54d47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -295,12 +295,13 @@ public class TestContainerAllocation {
int containerVersion, NodeId nodeId, String appSubmitter,
Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext, String nodeLabelExp,
- ContainerType containerType, ExecutionType executionType) {
+ ContainerType containerType, ExecutionType executionType,
+ long allocationRequestId) {
numRetries++;
return super.createContainerToken(containerId, containerVersion,
nodeId, appSubmitter, capability, priority, createTime,
logAggregationContext, nodeLabelExp, containerType,
- executionType);
+ executionType, allocationRequestId);
}
};
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/58] [abbrv] hadoop git commit: YARN-6626. Embed REST API service
into RM. Contributed by Eric Yang
Posted by ji...@apache.org.
YARN-6626. Embed REST API service into RM. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e677fa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e677fa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e677fa0
Branch: refs/heads/yarn-native-services
Commit: 9e677fa05c6a822cf034bffd25bbee0bad84084b
Parents: 8851209
Author: Jian He <ji...@apache.org>
Authored: Thu Sep 28 16:29:22 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:17 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 2 ++
.../hadoop/yarn/service/webapp/ApiServer.java | 7 ++++
.../hadoop-yarn-services-core/pom.xml | 6 ----
.../yarn/service/api/records/Artifact.java | 8 +++++
.../yarn/service/api/records/ConfigFile.java | 8 +++++
.../service/api/records/ReadinessCheck.java | 8 +++++
.../src/main/resources/yarn-default.xml | 8 +++++
.../server/resourcemanager/webapp/RMWebApp.java | 19 +++++++++++
.../site/markdown/yarn-service/QuickStart.md | 34 +++++++++++++++++---
9 files changed, 89 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 640e86e..4799137 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -336,6 +336,8 @@ public class YarnConfiguration extends Configuration {
public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
+ "webapp.ui2.war-file-path";
+ public static final String YARN_API_SERVICES_ENABLE = "yarn."
+ + "webapp.api-service.enable";
public static final String RM_RESOURCE_TRACKER_ADDRESS =
RM_PREFIX + "resource-tracker.address";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index e8286ef..89b020d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.yarn.service.webapp;
+import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.VersionInfo;
@@ -57,6 +58,12 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
@Singleton
@Path(CONTEXT_ROOT)
public class ApiServer {
+
+ @Inject
+ public ApiServer(Configuration conf) {
+ super();
+ }
+
private static final Logger LOG =
LoggerFactory.getLogger(ApiServer.class);
private static Configuration YARN_CONFIG = new YarnConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index fb07edc..205a64d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -245,12 +245,6 @@
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
index 4328c6f..ce062cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
@@ -23,6 +23,9 @@ import io.swagger.annotations.ApiModelProperty;
import java.io.Serializable;
import java.util.Objects;
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlType;
+
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
@@ -42,6 +45,11 @@ public class Artifact implements Serializable {
private String id = null;
+ /**
+ * Artifact Type. DOCKER, TARBALL or SERVICE
+ **/
+ @XmlType(name = "artifact_type")
+ @XmlEnum
public enum TypeEnum {
DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
index c1502c7..fb088e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -26,7 +26,10 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
@@ -45,6 +48,11 @@ import java.util.Objects;
public class ConfigFile implements Serializable {
private static final long serialVersionUID = -7009402089417704612L;
+ /**
+ * Config Type. XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported.
+ **/
+ @XmlType(name = "config_type")
+ @XmlEnum
public enum TypeEnum {
XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
"TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
index 1a25a4c..b25828f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -25,6 +25,9 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlType;
+
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -43,6 +46,11 @@ import org.apache.hadoop.classification.InterfaceStability;
public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L;
+ /**
+ * Type. HTTP and PORT
+ **/
+ @XmlType(name = "type")
+ @XmlEnum
public enum TypeEnum {
HTTP("HTTP"),
PORT("PORT");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index f4b2e61..e90d0f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -212,6 +212,14 @@
</property>
<property>
+ <description>
+ Enable services rest api on ResourceManager.
+ </description>
+ <name>yarn.webapp.api-service.enable</name>
+ <value>false</value>
+ </property>
+
+ <property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>${yarn.resourcemanager.hostname}:8031</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index 4e36665..bee9354 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -22,6 +22,8 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
import java.net.InetSocketAddress;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.RMHAUtils;
@@ -38,8 +40,12 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
*/
public class RMWebApp extends WebApp implements YarnWebParams {
+ private static final Log LOG =
+ LogFactory.getLog(RMWebApp.class.getName());
private final ResourceManager rm;
private boolean standby = false;
+ private final static String APISERVER =
+ "org.apache.hadoop.yarn.service.webapp.ApiServer";
public RMWebApp(ResourceManager rm) {
this.rm = rm;
@@ -53,6 +59,19 @@ public class RMWebApp extends WebApp implements YarnWebParams {
bind(RMWebApp.class).toInstance(this);
if (rm != null) {
+ boolean enableServiceApi = rm.getConfig()
+ .getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false);
+ if (enableServiceApi) {
+ try {
+ // Use reflection here to load ApiServer class,
+ // this is done to avoid creating cyclic dependency
+ // between maven projects.
+ Class<?> apiServer = Class.forName(APISERVER);
+ bind(apiServer);
+ } catch (ClassNotFoundException e) {
+ LOG.warn("ApiServer REST API is not activated.");
+ }
+ }
bind(ResourceManager.class).toInstance(rm);
}
route("/", RmController.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index 15df0cd..afcd5f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -101,17 +101,41 @@ yarn service destroy ${SERVICE_NAME}
```
## Manage services on YARN via REST API
-Below steps walk you through deploying services on YARN via REST API.
- Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
-### Start API-Server for deploying services on YARN
-API server is the service that sits in front of YARN ResourceManager and lets users submit their API specs via HTTP.
+
+YARN API Server REST API can be activated in two modes: embedded or standalone.
+
+### Start Embedded API-Server as part of ResourceManager
+For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
+
+```
+ <property>
+ <description>
+ Enable services rest api on ResourceManager.
+ </description>
+ <name>yarn.webapp.api-service.enable</name>
+ <value>true</value>
+ </property>
+```
+
+Services can be deployed on YARN through the ResourceManager web endpoint.
+
+### Start Standalone API-Server for deploying services on YARN
+API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP.
```
yarn --daemon start apiserver
```
The above command starts the API Server on the localhost at port 9191 by default.
+Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
+
### Deploy a service
-POST the aforementioned example service definition to the api-server endpoint:
+
+POST the aforementioned example service definition to the ResourceManager api-server endpoint:
+```
+POST http://localhost:8088/ws/v1/services
+```
+
+Or standalone API server:
```
POST http://localhost:9191/ws/v1/services
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/58] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in
yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
deleted file mode 100644
index f56139a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
+++ /dev/null
@@ -1,606 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# YARN Simplified API layer for services
-
-## Overview
-Bringing a new service on YARN today is not a simple experience. The APIs of
-existing frameworks are either too low level (native YARN), require writing
-new code (for frameworks with programmatic APIs) or writing a complex spec
-(for declarative frameworks). In addition to building critical building blocks
-inside YARN (as part of other efforts at
-[YARN-4692](https://issues.apache.org/jira/browse/YARN-4692)), there is a need for
-simplifying the user facing story for building services. Experience of projects
-like Apache Slider running real-life services like HBase, Storm, Accumulo,
-Solr etc, gives us some very good insights on how simplified APIs for services
-should look like.
-
-To this end, we should look at a new simple-services API layer backed by REST
-interfaces. This API can be used to create and manage the lifecycle of YARN
-services. Services here can range from simple single-component service to
-complex multi-component assemblies needing orchestration.
-[YARN-4793](https://issues.apache.org/jira/browse/YARN-4793) tracks this
-effort.
-
-This document spotlights on this specification. In most of the cases, the
-application owner will not be forced to make any changes to their applications.
-This is primarily true if the application is packaged with containerization
-technologies like docker. Irrespective of how complex the application is,
-there will be hooks provided at appropriate layers to allow pluggable and
-customizable application behavior.
-
-
-### Version information
-Version: 1.0.0
-
-### License information
-License: Apache 2.0
-License URL: http://www.apache.org/licenses/LICENSE-2.0.html
-
-### URI scheme
-Host: host.mycompany.com
-
-BasePath: /ws/v1/
-
-Schemes: HTTP
-
-### Consumes
-
-* application/json
-
-
-### Produces
-
-* application/json
-
-
-## Paths
-### Create a service
-```
-POST /services
-```
-
-#### Description
-
-Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|BodyParameter|Service|Service request object|true|Service||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|202|The request to create a service is accepted|No Content|
-|400|Invalid service definition provided in the request body|No Content|
-|500|Failed to create a service|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### (TBD) List of services running in the cluster.
-```
-GET /services
-```
-
-#### Description
-
-Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|200|An array of services|Service array|
-|default|Unexpected error|ServiceStatus|
-
-
-### Get current version of the API server.
-```
-GET /services/version
-```
-
-#### Description
-
-Get current version of the API server.
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|200|Successful request|No Content|
-
-
-### Update a service or upgrade the binary version of the components of a running service
-```
-PUT /services/{service_name}
-```
-
-#### Description
-
-Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|PathParameter|service_name|Service name|true|string||
-|BodyParameter|Service|The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service|true|Service||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|204|Update or upgrade was successful|No Content|
-|404|Service does not exist|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### Destroy a service
-```
-DELETE /services/{service_name}
-```
-
-#### Description
-
-Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|PathParameter|service_name|Service name|true|string||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|204|Destroy was successful|No Content|
-|404|Service does not exist|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### Get details of a service.
-```
-GET /services/{service_name}
-```
-
-#### Description
-
-Return the details (including containers) of a running service
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|PathParameter|service_name|Service name|true|string||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|200|a service object|object|
-|404|Service does not exist|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### Flex a component's number of instances.
-```
-PUT /services/{service_name}/components/{component_name}
-```
-
-#### Description
-
-Set a component's desired number of instanes
-
-#### Parameters
-|Type|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|----|
-|PathParameter|service_name|Service name|true|string||
-|PathParameter|component_name|Component name|true|string||
-|BodyParameter|Component|The definition of a component which contains the updated number of instances.|true|Component||
-
-
-#### Responses
-|HTTP Code|Description|Schema|
-|----|----|----|
-|200|Flex was successful|No Content|
-|404|Service does not exist|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-## Definitions
-### Artifact
-
-Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|id|Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.|true|string||
-|type|Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).|false|enum (DOCKER, TARBALL, SERVICE)|DOCKER|
-|uri|Artifact location to support multiple artifact stores (optional).|false|string||
-
-
-### Component
-
-One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|name|Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.|true|string||
-|dependencies|An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.|false|string array||
-|readiness_check|Readiness check for this component.|false|ReadinessCheck||
-|artifact|Artifact of the component (optional). If not specified, the service level global artifact takes effect.|false|Artifact||
-|launch_command|The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).|false|string||
-|resource|Resource of this component (optional). If not specified, the service level global resource takes effect.|false|Resource||
-|number_of_containers|Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.|false|integer (int64)||
-|run_privileged_container|Run all containers of this component in privileged mode (YARN-4262).|false|boolean||
-|placement_policy|Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.|false|PlacementPolicy||
-|configuration|Config properties for this component.|false|Configuration||
-|quicklinks|A list of quicklink keys defined at the service level, and to be resolved by this component.|false|string array||
-
-
-### ConfigFile
-
-A config file that needs to be created and made available as a volume in a service component container.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|type|Config file in the standard format like xml, properties, json, yaml, template.|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, ENV, HADOOP_XML)||
-|dest_file|The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.|false|string||
-|src_file|This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.|false|string||
-|props|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
-
-
-### Configuration
-
-Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|properties|A blob of key-value pairs of common service properties.|false|object||
-|env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object||
-|files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array||
-
-
-### Container
-
-An instance of a running service container.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|id|Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.|false|string||
-|launch_time|The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.|false|string (date)||
-|ip|IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
-|hostname|Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
-|bare_host|The bare node or host in which the container is running, e.g. cn008.example.com.|false|string||
-|state|State of the container of a service.|false|ContainerState||
-|component_name|Name of the component that this container instance belongs to.|false|string||
-|resource|Resource used for this container.|false|Resource||
-|artifact|Artifact used for this container.|false|Artifact||
-|privileged_container|Container running in privileged mode or not.|false|boolean||
-
-
-### ContainerState
-
-The current state of the container of a service.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|state|enum of the state of the container|false|enum (INIT, STARTED, READY)||
-
-
-### PlacementPolicy
-
-Placement policy of an instance of a service. This feature is in the works in YARN-6592.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|label|Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.|false|string||
-
-
-### ReadinessCheck
-
-A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|type|E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).|true|enum (HTTP, PORT)||
-|props|A blob of key value pairs that will be used to configure the check.|false|object||
-|artifact|Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET|false|Artifact||
-
-
-### Resource
-
-Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string||
-|cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)||
-|memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string||
-
-
-### Service
-
-a service resource has the following attributes.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|name|A unique service name. If Registry DNS is enabled, the max length is 63 characters.|true|string||
-|id|A unique service id.|false|string||
-|artifact|Artifact of single-component service.|false|Artifact||
-|resource|Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.|false|Resource||
-|launch_command|The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.|false|string||
-|launch_time|The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.|false|string (date)||
-|number_of_containers|Number of containers for each component in the service. Each component can further override this service-level global default.|false|integer (int64)||
-|number_of_running_containers|In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.|false|integer (int64)||
-|lifetime|Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.|false|integer (int64)||
-|placement_policy|(TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.|false|PlacementPolicy||
-|components|Components of a service.|false|Component array||
-|configuration|Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.|false|Configuration||
-|containers|Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.|false|Container array||
-|state|State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.|false|ServiceState||
-|quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
-|queue|The YARN queue that this service should be submitted to.|false|string||
-
-
-### ServiceState
-
-The current state of a service.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, READY, STOPPED, FAILED)||
-
-
-### ServiceStatus
-
-The current status of a submitted service, returned as a response to the GET API.
-
-|Name|Description|Required|Schema|Default|
-|----|----|----|----|----|
-|diagnostics|Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.|false|string||
-|state|Service state.|false|ServiceState||
-|code|An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.|false|integer (int32)||
-
-
-
-## Examples
-
-### Create a simple single-component service with most attribute values as defaults
-POST URL - http://localhost:9191/ws/v1/services
-
-##### POST Request JSON
-```json
-{
- "name": "hello-world",
- "components" :
- [
- {
- "name": "hello",
- "number_of_containers": 1,
- "artifact": {
- "id": "nginx:latest",
- "type": "DOCKER"
- },
- "launch_command": "./start_nginx.sh",
- "resource": {
- "cpus": 1,
- "memory": "256"
- }
- }
- ]
-}
-```
-
-##### GET Response JSON
-GET URL - http://localhost:9191/ws/v1/services/hello-world
-
-Note, lifetime value of -1 means unlimited lifetime.
-
-```json
-{
- "name": "hello-world",
- "id": "application_1503963985568_0002",
- "lifetime": -1,
- "components": [
- {
- "name": "hello",
- "dependencies": [],
- "resource": {
- "cpus": 1,
- "memory": "256"
- },
- "configuration": {
- "properties": {},
- "env": {},
- "files": []
- },
- "quicklinks": [],
- "containers": [
- {
- "id": "container_e03_1503963985568_0002_01_000001",
- "ip": "10.22.8.143",
- "hostname": "myhost.local",
- "state": "READY",
- "launch_time": 1504051512412,
- "bare_host": "10.22.8.143",
- "component_name": "hello-0"
- },
- {
- "id": "container_e03_1503963985568_0002_01_000002",
- "ip": "10.22.8.143",
- "hostname": "myhost.local",
- "state": "READY",
- "launch_time": 1504051536450,
- "bare_host": "10.22.8.143",
- "component_name": "hello-1"
- }
- ],
- "launch_command": "./start_nginx.sh",
- "number_of_containers": 1,
- "run_privileged_container": false
- }
- ],
- "configuration": {
- "properties": {},
- "env": {},
- "files": []
- },
- "quicklinks": {}
-}
-
-```
-### Update to modify the lifetime of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
-
-##### PUT Request JSON
-
-Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
-
-```json
-{
- "lifetime": 3600
-}
-```
-### Stop a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
-
-##### PUT Request JSON
-```json
-{
- "state": "STOPPED"
-}
-```
-
-### Start a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
-
-##### PUT Request JSON
-```json
-{
- "state": "STARTED"
-}
-```
-
-### Update to flex up/down the no of containers (instances) of a component of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
-
-##### PUT Request JSON
-```json
-{
- "name": "hello",
- "number_of_containers": 3
-}
-```
-
-### Destroy a service
-DELETE URL - http://localhost:9191/ws/v1/services/hello-world
-
-***
-
-### Create a complicated service - HBase
-POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
-
-##### POST Request JSON
-
-```json
-{
- "name": "hbase-app-1",
- "lifetime": "3600",
- "components": [
- {
- "name": "hbasemaster",
- "number_of_containers": 1,
- "artifact": {
- "id": "hbase:latest",
- "type": "DOCKER"
- },
- "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
- "resource": {
- "cpus": 1,
- "memory": "2048"
- },
- "configuration": {
- "env": {
- "HBASE_LOG_DIR": "<LOG_DIR>"
- },
- "files": [
- {
- "type": "XML",
- "dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
- "fs.defaultFS": "${CLUSTER_FS_URI}"
- }
- },
- {
- "type": "XML",
- "dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
- "hbase.cluster.distributed": "true",
- "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
- "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
- "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
- "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
- "hbase.master.info.port": "16010"
- }
- }
- ]
- }
- },
- {
- "name": "regionserver",
- "number_of_containers": 3,
- "unique_component_support": "true",
- "artifact": {
- "id": "hbase:latest",
- "type": "DOCKER"
- },
- "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
- "resource": {
- "cpus": 1,
- "memory": "2048"
- },
- "configuration": {
- "env": {
- "HBASE_LOG_DIR": "<LOG_DIR>"
- },
- "files": [
- {
- "type": "XML",
- "dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
- "fs.defaultFS": "${CLUSTER_FS_URI}"
- }
- },
- {
- "type": "XML",
- "dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
- "hbase.cluster.distributed": "true",
- "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
- "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
- "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
- "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
- "hbase.master.info.port": "16010",
- "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
- }
- }
- ]
- }
- }
- ],
- "quicklinks": {
- "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
- "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
- }
-}
-```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
deleted file mode 100644
index a927118..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
+++ /dev/null
@@ -1,144 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# YARN DNS Server
-
-<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-
-## Introduction
-
-The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
-
-1. **Exposing existing service-discovery information via DNS** - Information provided in
-the current YARN service registry’s records will be converted into DNS entries, thus
-allowing users to discover information about YARN applications using standard DNS
-client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
-number for services).
-2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
-standard DNS lookups. Given the availability of the records via DNS, container
-name-based communication will be facilitated (e.g. ‘curl
-http://myContainer.myDomain.com/endpoint’).
-
-## Service Properties
-
-The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
-
-The following core functions are supported by the DNS-Server:
-
-### Functional properties
-
-1. Supports creation of DNS records for end-points of the deployed YARN applications
-2. Record names remain unchanged during restart of containers and/or applications
-3. Supports reverse lookups (name based on IP).
-4. Supports security using the standards defined by The Domain Name System Security
-Extensions (DNSSEC)
-5. Highly available
-6. Scalable - The service provides the responsiveness (e.g. low-latency) required to
-respond to DNS queries (timeouts yield attempts to invoke other configured name
-servers).
-
-### Deployment properties
-
-1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as
-a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a
-primary DNS server and does not forward requests to other servers.
-2. The DNS Server exposes a port that can receive both TCP and UDP requests per
-DNS standards. The default port for DNS protocols is in a restricted, administrative port
-range (53), so the port is configurable for deployments in which the service may
-not be managed via an administrative account.
-
-## DNS Record Name Structure
-
-The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see “Preferred Name Syntax” in RFC 1035):
-
-* **domain** - the name of the cluster DNS domain. This name is provided as a
-configuration property. In addition, it is this name that is configured at a parent DNS
-server as the zone name for the defined yDNS zone (the zone for which the parent DNS
-server will forward requests to yDNS). E.g. yarncluster.com
-* **username** - the name of the application deployer. This name is the simple short-name (for
-e.g. the primary component of the Kerberos principal) associated with the user launching
-the application. As the username is one of the elements of DNS names, it is expected
-that this also confirms DNS name conventions (RFC 1035 linked above), so special translation is performed for names with special characters like hyphens and spaces.
-* **application name** - the name of the deployed YARN application. This name is inferred
-from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
-names. This obviously mandates certain uniqueness properties on application names.
-* **container id** - the YARN assigned ID to a container (e.g.
-container_e3741_1454001598828_01_000004)
-* **component name** - the name assigned to the deployed component (for e.g. a master
-component). A component is a distributed element of an application or service that is
-launched in a YARN container (e.g. an HBase master). One can imagine multiple
-components within an application. A component name is not yet a first class concept in
-YARN, but is a very useful one that we are introducing here for the sake of yDNS
-entries. Many frameworks like MapReduce, Slider already have component names
-(though, as mentioned, they are not yet supported in YARN in a first class fashion).
-* **api** - the api designation for the exposed endpoint
-
-### Notes about DNS Names
-
-* In most instances, the DNS names can be easily distinguished by the number of
-elements/labels that compose the name. The cluster’s domain name is always the last
-element. After that element is parsed out, reading from right to left, the first element
-maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
-“container” or suffix such as “api”. For example, an endpoint published as a
-management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
-* Unique application name (per user) is not currently supported/guaranteed by YARN, but
-it is supported by frameworks such as Apache Slider. The yDNS service currently
-leverages the last element of the ZK path entry for the application as an
-application name. These application names have to be unique for a given user.
-
-## DNS Server Functionality
-
-The primary functions of the DNS service are illustrated in the following diagram:
-
-
-
-
-### DNS record creation
-The following figure illustrates at slightly greater detail the DNS record creation and registration sequence (NOTE: service record updates would follow a similar sequence of steps,
-distinguished only by the different event type):
-
-
-
-### DNS record removal
-Similarly, record removal follows a similar sequence
-
-
-
-(NOTE: The DNS Zone requires a record as an argument for the deletion method, thus
-requiring similar parsing logic to identify the specific records that should be removed).
-
-### DNS Service initialization
-* The DNS service initializes both UDP and TCP listeners on a configured port. As
-noted above, the default port of 53 is in a restricted range that is only accessible to an
-account with administrative privileges.
-* Subsequently, the DNS service listens for inbound DNS requests. Those requests are
-standard DNS requests from users or other DNS servers (for example, DNS servers that have the
-YARN DNS service configured as a forwarder).
-
-## Configuration
-The YARN DNS server reads its configuration properties from the yarn-site.xml file. The following are the DNS associated configuration properties:
-
-| Name | Description |
-| ------------ | ------------- |
-| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
-| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
-| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
-| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 53. However, since that port falls in a administrator-only range, typical deployments may need to specify an alternate port. |
-| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
-| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
-| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
-| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
-| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
-| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
-| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
deleted file mode 100644
index e6a4e91..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
+++ /dev/null
@@ -1,107 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Introduction: YARN Native Services
-
-## Overview
-YARN Native Services provides first class framework support and APIs to host long running services natively in YARN. In addition to launching services, the new APIs support performing lifecycle management operations, such as flex service components up/down, manage lifetime, upgrade the service to a newer version, and stop/restart/delete the service.
-
-The native services capabilities are built on the existing low-level resource management API provided by YARN that can support any type of application. Other application frameworks like Hadoop MapReduce already expose higher level APIs that users can leverage to run applications on top of YARN. With the advent of containerization technologies like Docker, providing first class support and APIs for long running services at the framework level made sense.
-
-Relying on a framework has the advantage of exposing a simpler usage model to the user by enabling service configuration and launch through specification (without writing new code), as well as hiding complex low-level details including state management and fault-tolerance etc. Users/operators of existing services typically like to avoid modifying an existing service to be aware of YARN. With first class support capable of running a single Docker image as well as complex assemblies comprised of multiple Docker images, there is no need for service owners to be aware of YARN. Developers of new services do not have to worry about YARN internals and only need to focus on containerization of their service(s).
-
-## First class support for services
-In order to natively provide first class support for long running services, several new features and improvements have been made at the framework level.
-
-### Incorporate Apache Slider into Apache YARN
-Apache Slider, which existed as a separate incubator project has been merged into YARN to kick start the first class support. Apache Slider is a universal Application Master (AM) which had several key features built in - fault tolerance of service containers and AM, work-preserving AM restarts, service logs management, service management like flex up/down, stop/start, and rolling upgrade to newer service versions, etc. Of course lot more work has been done on top of what Apache Slider brought in, details of which follow.
-
-### Native Services API
-A significant effort has gone into simplifying the user facing story for building services. In the past, bringing a new service to YARN was not a pleasant experience. The APIs of existing frameworks are either too low-level (native YARN), require writing new code (for frameworks with programmatic APIs) or require writing a complex spec (for declarative frameworks).
-
-The new REST APIs are very simple to use. The REST layer acts as a single point of entry for creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex, multi-component applications needing special orchestration needs.
-
-Plan is to make this a unified REST based entry point for other important features like resource-profile management ([YARN-3926](https://issues.apache.org/jira/browse/YARN-4793)), package-definitions' lifecycle-management and service-discovery ([YARN-913](https://issues.apache.org/jira/browse/YARN-913)/[YARN-4757](https://issues.apache.org/jira/browse/YARN-4757)).
-
-### Native Services Discovery
-The new discovery solution exposes the registry information through a more generic and widely used mechanism: DNS. Service Discovery via DNS uses the well-known DNS interfaces to browse the network for services. Having the registry information exposed via DNS simplifies the life of services.
-
-The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface. In practice, this made it very difficult for wiring up existing clients and services. For e.g., dynamic configuration of dependent endpoints of a service was not easy to implement using the registry-read mechanisms, **without** code-changes to existing services. These are solved by the DNS based service discovery.
-
-### Scheduling
-[YARN-6592](https://issues.apache.org/jira/browse/YARN-6592) covers a host of scheduling features that are useful for short-running applications and services alike. Below, are a few very important YARN core features that help schedule services better. Without these, running services on YARN is a hassle.
-
-* Affinity (TBD)
-* Anti-affinity (TBD)
-* Gang scheduling (TBD)
-* Malleable container sizes ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197))
-
-### Resource Profiles
-YARN always had support for memory as a resource, inheriting it from Hadoop-(1.x)’s MapReduce platform. Later support for CPU as a resource ([YARN-2](https://issues.apache.org/jira/browse/YARN-2)/[YARN-3](https://issues.apache.org/jira/browse/YARN-3)) was added. Multiple efforts added support for various other resource-types in YARN such as disk ([YARN-2139](https://issues.apache.org/jira/browse/YARN-2139)), and network ([YARN-2140](https://issues.apache.org/jira/browse/YARN-2140)), specifically benefiting long running services.
-
-In many systems outside of YARN, users are already accustomed to specifying their desired ‘box’ of requirements where each box comes with a predefined amount of each resources. Admins would define various available box-sizes (small, medium, large etc) and users would pick the ones they desire and everybody is happy. In [YARN-3926](https://issues.apache.org/jira/browse/YARN-3926), YARN introduces Resource Profiles which extends the YARN resource model for easier resource-type management and profiles. This helps in two ways - the system can schedule applications better and it can perform intelligent over-subscription of resources where applicable.
-
-Resource profiles are all the more important for services since -
-* Similar to short running apps, you don’t have to fiddle with varying resource-requirements for each container type
-* Services usually end up planning for peak usages, leaving a lot of possibility of barren utilization
-
-### Special handling of preemption and container reservations
-TBD.
-
-Preemption and reservation of long running containers have different implications from regular ones. Preemption of resources in YARN today works by killing of containers. For long-lived services this is unacceptable. Also, scheduler should avoid allocating long running containers on borrowed resources. [YARN-4724](https://issues.apache.org/jira/browse/YARN-4724) will address some of these special recognition of service containers.
-
-### Container auto-restarts
-If a service container dies, expiring container's allocation and releasing the allocation is undesirable in many cases. Long running containers may exit for various reasons, crash and need to restart but forcing them to go through the complete scheduling cycle, resource localization, etc. is both unnecessary and expensive.
-
-Services can enable app-specific policies to prevent NodeManagers to automatically restart containers. [YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails.
-
-### Container allocation re-use for application upgrades
-TBD.
-
-Auto-restart of containers will support upgrade of service containers without reclaiming the resources first. During an upgrade, with multitude of other applications running in the system, giving up and getting back resources allocated to the service is hard to manage. Node-Labels help this cause but are not straight-forward to use to address the app-specific use-cases. The umbrella [YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) along with [YARN-5620](https://issues.apache.org/jira/browse/YARN-5620) and [YARN-4470](https://issues.apache.org/jira/browse/YARN-4470) will take care of this.
-
-### Dynamic Configurations
-Most production-level services require dynamic configurations to manage and simplify their lifecycle. Container’s resource size, local/work dirs and log-dirs are the most basic information services need. Service's endpoint details (host/port), their inter-component dependencies, health-check endpoints, etc. are all critical to the success of today's real-life services.
-
-### Resource re-localization for reconfiguration/upgrades
-TBD
-
-### Service Registry
-TBD
-
-### Service persistent storage and volume support
-TBD
-
-### Packaging
-TBD
-
-### Container image registry (private, public and hybrid)
-TBD
-
-### Container image management and APIs
-TBD
-
-### Container image storage
-TBD
-
-### Monitoring
-TBD
-
-### Metrics
-TBD
-
-### Service Logs
-TBD
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
new file mode 100644
index 0000000..7b62c36
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
@@ -0,0 +1,77 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Concepts
+This document describes some key concepts and features that makes YARN as a first-class platform in order to natively support long running services on YARN.
+
+### Service Framework (ApplicationMaster) on YARN
+A container orchestration framework is implemented to help deploying services on YARN. In a nutshell, the framework is an ApplicationMaster that
+requests containers from ResourceManager based on service definition provided by the user and launch the containers across the cluster adhering to placement policies.
+It also does all the heavy lifting work such as resolving the service definition and configurations, managing component life cycles such as automatically restarting
+failed containers, monitoring components' healthiness and readiness, ensuring dependency start order across components, flexing up/down components,
+upgrading components etc. The end goal of the framework is to make sure the service is up and running as the state that user desired.
+
+
+### A Restful API-Server for deploying/managing services on YARN
+A restful API server is developed to allow users to deploy/manage their services on YARN via a simple JSON spec. This avoids users
+from dealing with the low-level APIs, writing complex code to bring their services onto YARN. The REST layer acts as a unified REST based entry for
+creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex,
+multi-component applications needing special orchestration needs. Please refer to this [API doc](YarnServiceAPI.md) for detailed API documentations.
+
+The API-server is stateless, which means users can simply spin up multiple instances, and have a load balancer fronting them to
+support HA, distribute the load etc.
+
+### Service Discovery
+A DNS server is implemented to enable discovering services on YARN via the standard mechanism: DNS lookup.
+The DNS server essentially exposes the information in YARN service registry by translating them into DNS records such as A record and SRV record.
+Clients can discover the IPs of containers via standard DNS lookup.
+The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult
+to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [DNS doc](ServiceDiscovery.md)
+for more details.
+
+### Scheduling
+
+A host of scheduling features are being developed to support long running services.
+
+* Affinity and anti-affinity scheduling across containers ([YARN-6592](https://issues.apache.org/jira/browse/YARN-6592)).
+* Container resizing ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197))
+* Special handling of container preemption/reservation for services
+
+### Container auto-restarts
+
+[YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails.
+The service REST API provides users a way to enable NodeManager to automatically restart the container if it fails.
+The advantage is that it avoids the entire cycle of releasing the failed containers, re-asking new containers, re-do resource localizations and so on, which
+greatly minimizes container downtime.
+
+
+### Container in-place upgrade
+
+[YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) aims to support upgrading containers in-place, that is, without losing the container allocations.
+It opens up a few APIs in NodeManager to allow ApplicationMasters to upgrade their containers via a simple API call.
+Under the hood, NodeManager does below steps:
+* Downloading the new resources such as jars, docker container images, new configurations.
+* Stop the old container.
+* Start the new container with the newly downloaded resources.
+
+At the time of writing this document, core changes are done but the feature is not usable end-to-end.
+
+### Resource Profiles
+
+In [YARN-3926](https://issues.apache.org/jira/browse/YARN-3926), YARN introduces Resource Profiles which extends the YARN resource model for easier
+resource-type management and profiles.
+It primarily solves two problems:
+* Make it easy to support new resource types such as network bandwith([YARN-2140](https://issues.apache.org/jira/browse/YARN-2140)), disks([YARN-2139](https://issues.apache.org/jira/browse/YARN-2139)).
+ Under the hood, it unifies the scheduler codebase to essentially parameterize the resource types.
+* User can specify the container resource requirement by a profile name, rather than fiddling with varying resource-requirements for each resource type.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
new file mode 100644
index 0000000..3dd4d8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -0,0 +1,58 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN Service
+## Overview
+Yarn Service framework provides first class support and APIs to host long running services natively in YARN.
+In a nutshell, it serves as a container orchestration platform for managing containerized services on YARN. It supports both docker container
+and traditional process based containers in YARN.
+
+The responsibility of this framework includes performing configuration resolutions and mounts,
+lifecycle management such as stop/start/delete the service, flexing service components up/down, rolling upgrades services on YARN, monitoring services' healthiness and readiness and more.
+
+The yarn-service framework primarily includes below components:
+
+* A core framework (ApplicationMaster) running on YARN to serve as a container orchestrator, being responsible for all service lifecycle managements.
+* A restful API-server to for users to interact with YARN to deploy/manage their services via a simple JSON spec.
+* A DNS server backed by YARN service registry to enable discovering services on YARN by the standard DNS lookup.
+
+## Why should I try YARN Service framework?
+
+YARN Service framework makes it easy to bring existing services onto YARN.
+It hides all the complex low-level details of application management and relieves
+users from forced into writing new code. Developers of new services do not have
+to worry about YARN internals and only need to focus on containerization of their
+service(s).
+
+Further, another huge win of this feature is that now you can enable both
+traditional batch processing jobs and long running services in a single platform!
+The benefits of combining these workloads are two-fold:
+
+* Greatly simplify the cluster operations as you have only a single cluster to deal with.
+* Making both batch jobs and services share a cluster can greatly improve resource utilization.
+
+## How do I get started?
+
+*`This feature is in alpha state`* and so APIs, command lines are subject to change. We will continue to update the documents over time.
+
+[QuickStart](QuickStart.md) shows a quick tutorial that walks you through simple steps to deploy a service on YARN.
+
+## How do I get my hands dirty?
+
+* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN.
+* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN.
+* [Service Discovery](ServiceDiscovery.md): Deep dives into the YARN DNS internals
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
new file mode 100644
index 0000000..327566b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -0,0 +1,218 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Quick Start
+
+This document describes how to deploy services on YARN using the YARN Service framework.
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+## Start HDFS and YARN components
+
+ Start all the hadoop components HDFS, YARN as usual.
+
+
+## Example service
+Below is a simple service definition that launches sleep containers on YARN by writing a simple spec file and without writing any code.
+
+```
+{
+ "name": "sleeper-service",
+ "components" :
+ [
+ {
+ "name": "sleeper",
+ "number_of_containers": 1,
+ "launch_command": "sleep 900000",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+
+For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
+
+## Manage services on YARN via CLI
+Below steps walk you through deploying a services on YARN using CLI.
+Refer to [Yarn Commands](../YarnCommands.md) for the full list of commands and options.
+### Deploy a service
+```
+yarn service create --file ${PATH_TO_SERVICE_DEF_FILE}
+```
+Params:
+- SERVICE_NAME: The name of the service. Note that this needs to be unique across all running services.
+- PATH_TO_SERVICE_DEF: The path to the service definition file in JSON format.
+
+For example:
+```
+yarn service create --file /path/to/local/sleeper.json
+```
+
+### Flex a component of a service
+Increase or decrease the number of containers for a component.
+```
+yarn service flex ${SERVICE_NAME} --component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS}
+```
+For example, for a service named `sleeper-service`:
+
+Set the `sleeper` component to `2` containers (absolute number).
+
+```
+yarn service flex sleeper-service --component sleeper 2
+```
+
+### Stop a service
+Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
+```
+yarn service stop ${SERVICE_NAME}
+```
+
+### Restart a stopped service
+Restarting a stopped service is easy - just call start!
+```
+yarn service start ${SERVICE_NAME}
+```
+
+### Destroy a service
+In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
+```
+yarn service destroy ${SERVICE_NAME}
+```
+
+## Manage services on YARN via REST API
+Below steps walk you through deploying services on YARN via REST API.
+ Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
+### Start API-Server for deploying services on YARN
+API server is the service that sits in front of YARN ResourceManager and lets users submit their API specs via HTTP.
+```
+ yarn --daemon start apiserver
+ ```
+The above command starts the API Server on the localhost at port 9191 by default.
+
+### Deploy a service
+POST the aforementioned example service definition to the api-server endpoint:
+```
+POST http://localhost:9191/ws/v1/services
+```
+
+### Get a service status
+```
+GET http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+```
+
+### Flex a component of a service
+```
+PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME}
+```
+`PUT` Request body:
+```
+{
+ "name": "${COMPONENT_NAME}",
+ "number_of_containers": ${COUNT}
+}
+```
+For example:
+```
+{
+ "name": "sleeper",
+ "number_of_containers": 2
+}
+```
+
+### Stop a service
+Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
+
+```
+PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+```
+
+`PUT` Request body:
+```
+{
+ "name": "${SERVICE_NAME}",
+ "state": "STOPPED"
+}
+```
+
+### Restart a stopped service
+Restarting a stopped service is easy.
+
+```
+PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+```
+
+`PUT` Request body:
+```
+{
+ "name": "${SERVICE_NAME}",
+ "state": "STARTED"
+}
+```
+### Destroy a service
+In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
+```
+DELETE http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+```
+
+## Services UI with YARN UI2 and Timeline Service v2
+A new `service` tab is added in the YARN UI2 specially to show YARN Services in a first class manner.
+The services framework posts the data into TimelineService and the `service` UI reads data from TimelineService to render its content.
+
+### Enable Timeline Service v2
+Please refer to [TimeLineService v2 doc](../TimelineServiceV2.md) for how to enable Timeline Service v2.
+
+### Enable new YARN UI
+
+Set below config in `yarn-site.xml` and start ResourceManager.
+If you are building from source code, make sure you use `-Pyarn-ui` in the `mvn` command - this will generate the war file for the new YARN UI.
+```
+ <property>
+ <description>To enable RM web ui2 application.</description>
+ <name>yarn.webapp.ui2.enable</name>
+ <value>true</value>
+ </property>
+```
+
+## Service Discovery with YARN DNS
+YARN Service framework comes with a DNS server (backed by YARN Service Registry) which enables DNS based discovery of services deployed on YARN.
+That is, user can simply access their services in a well-defined naming format as below:
+
+```
+${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}
+```
+For example, in a cluster whose domain name is `yarncluster` (as defined by the `hadoop.registry.dns.domain-name` in `yarn-site.xml`), a service named `hbase` deployed by user `dev`
+with two components `hbasemaster` and `regionserver` can be accessed as below:
+
+This URL points to the usual hbase master UI
+```
+http://hbasemaster-0.hbase.dev.yarncluster:16010/master-status
+```
+
+
+Note that YARN service framework assigns COMPONENT_INSTANCE_NAME for each container in a sequence of monotonically increasing integers. For example, `hbasemaster-0` gets
+assigned `0` since it is the first and only instance for the `hbasemaster` component. In case of `regionserver` component, it can have multiple containers
+ and so be named as such: `regionserver-0`, `regionserver-1`, `regionserver-2` ... etc
+
+`Disclaimer`: The DNS implementation is still experimental. It should not be used as a fully-functional corporate DNS.
+
+### Start the DNS server
+By default, the DNS runs on non-privileged port `5353`.
+If it is configured to use the standard privileged port `53`, the DNS server needs to be run as root:
+```
+sudo su - -c "yarn org.apache.hadoop.registry.server.dns.RegistryDNSServer > /${HADOOP_LOG_FOLDER}/registryDNS.log 2>&1 &" root
+```
+Please refer to [YARN DNS doc](ServicesDiscovery.md) for the full list of configurations.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
new file mode 100644
index 0000000..6318a07
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
@@ -0,0 +1,150 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN DNS Server
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+## Introduction
+
+The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
+
+1. **Exposing existing service-discovery information via DNS** - Information provided in
+the current YARN service registry’s records will be converted into DNS entries, thus
+allowing users to discover information about YARN applications using standard DNS
+client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
+number for services).
+2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
+standard DNS lookups. Given the availability of the records via DNS, container
+name-based communication will be facilitated (e.g. ‘curl
+http://myContainer.myDomain.com/endpoint’).
+
+## Service Properties
+
+The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
+
+The following core functions are supported by the DNS-Server:
+
+### Functional properties
+
+1. Supports creation of DNS records for end-points of the deployed YARN applications
+2. Record names remain unchanged during restart of containers and/or applications
+3. Supports reverse lookups (name based on IP). Note, this works only for Docker containers.
+4. Supports security using the standards defined by The Domain Name System Security
+Extensions (DNSSEC)
+5. Highly available
+6. Scalable - The service provides the responsiveness (e.g. low-latency) required to
+respond to DNS queries (timeouts yield attempts to invoke other configured name
+servers).
+
+### Deployment properties
+
+1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as
+a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a
+primary DNS server and does not forward requests to other servers.
+2. The DNS Server exposes a port that can receive both TCP and UDP requests per
+DNS standards. The default port for DNS protocols is in a restricted, administrative port
+range (5353), so the port is configurable for deployments in which the service may
+not be managed via an administrative account.
+
+## DNS Record Name Structure
+
+The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see “Preferred Name Syntax” in RFC 1035):
+
+* **domain** - the name of the cluster DNS domain. This name is provided as a
+configuration property. In addition, it is this name that is configured at a parent DNS
+server as the zone name for the defined yDNS zone (the zone for which the parent DNS
+server will forward requests to yDNS). E.g. yarncluster.com
+* **username** - the name of the application deployer. This name is the simple short-name (for
+e.g. the primary component of the Kerberos principal) associated with the user launching
+the application. As the username is one of the elements of DNS names, it is expected
+that this also confirms DNS name conventions (RFC 1035 linked above), so special translation is performed for names with special characters like hyphens and spaces.
+* **application name** - the name of the deployed YARN application. This name is inferred
+from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
+names. This obviously mandates certain uniqueness properties on application names.
+* **container id** - the YARN assigned ID to a container (e.g.
+container_e3741_1454001598828_01_000004)
+* **component name** - the name assigned to the deployed component (for e.g. a master
+component). A component is a distributed element of an application or service that is
+launched in a YARN container (e.g. an HBase master). One can imagine multiple
+components within an application. A component name is not yet a first class concept in
+YARN, but is a very useful one that we are introducing here for the sake of yDNS
+entries. Many frameworks like MapReduce, Slider already have component names
+(though, as mentioned, they are not yet supported in YARN in a first class fashion).
+* **api** - the api designation for the exposed endpoint
+
+### Notes about DNS Names
+
+* In most instances, the DNS names can be easily distinguished by the number of
+elements/labels that compose the name. The cluster’s domain name is always the last
+element. After that element is parsed out, reading from right to left, the first element
+maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
+“container” or suffix such as “api”. For example, an endpoint published as a
+management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
+* Unique application name (per user) is not currently supported/guaranteed by YARN, but
+it is supported by frameworks such as Apache Slider. The yDNS service currently
+leverages the last element of the ZK path entry for the application as an
+application name. These application names have to be unique for a given user.
+
+## DNS Server Functionality
+
+The primary functions of the DNS service are illustrated in the following diagram:
+
+
+
+### DNS record creation
+The following figure illustrates at slightly greater detail the DNS record creation and registration sequence (NOTE: service record updates would follow a similar sequence of steps,
+distinguished only by the different event type):
+
+
+
+### DNS record removal
+Similarly, record removal follows a similar sequence
+
+
+
+(NOTE: The DNS Zone requires a record as an argument for the deletion method, thus
+requiring similar parsing logic to identify the specific records that should be removed).
+
+### DNS Service initialization
+* The DNS service initializes both UDP and TCP listeners on a configured port. As
+noted above, the default port of 5353 is in a restricted range that is only accessible to an
+account with administrative privileges.
+* Subsequently, the DNS service listens for inbound DNS requests. Those requests are
+standard DNS requests from users or other DNS servers (for example, DNS servers that have the
+YARN DNS service configured as a forwarder).
+
+## Start the DNS Server
+By default, the DNS runs on non-privileged port `5353`.
+If it is configured to use the standard privileged port `53`, the DNS server needs to be run as root:
+```
+sudo su - -c "yarn org.apache.hadoop.registry.server.dns.RegistryDNSServer > /${HADOOP_LOG_FOLDER}/registryDNS.log 2>&1 &" root
+```
+
+## Configuration
+The YARN DNS server reads its configuration properties from the yarn-site.xml file. The following are the DNS associated configuration properties:
+
+| Name | Description |
+| ------------ | ------------- |
+| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
+| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
+| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
+| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 5353. However, since that port falls in a administrator-only range, typical deployments may need to specify an alternate port. |
+| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
+| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
+| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
+| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
+| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
+| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
+| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/58] [abbrv] hadoop git commit: YARN-7161. Fix special chars in
DNS documentation. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-7161. Fix special chars in DNS documentation. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9ca13e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9ca13e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9ca13e3
Branch: refs/heads/yarn-native-services
Commit: b9ca13e34bc8a14787e94e9b70ae0aa249f00954
Parents: eed9e3e
Author: Jian He <ji...@apache.org>
Authored: Thu Sep 7 18:17:02 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:15 2017 -0800
----------------------------------------------------------------------
.../native-services/NativeServicesDiscovery.md | 87 ++++++++++----------
1 file changed, 45 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9ca13e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
index fa54a09..a927118 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
@@ -13,39 +13,42 @@
-->
# YARN DNS Server
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
## Introduction
The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
-1. **Exposing existing service discovery information via DNS** - Information provided in
+1. **Exposing existing service-discovery information via DNS** - Information provided in
the current YARN service registry’s records will be converted into DNS entries, thus
allowing users to discover information about YARN applications using standard DNS
client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
number for services).
-2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
+2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
standard DNS lookups. Given the availability of the records via DNS, container
-namebased communication will be facilitated (e.g. ‘curl
+name-based communication will be facilitated (e.g. ‘curl
http://myContainer.myDomain.com/endpoint’).
## Service Properties
The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
-The following core functions are supported by the DNS Server:
+The following core functions are supported by the DNS-Server:
-###Functional properties
+### Functional properties
-1. Supports creation of DNS records for endpoints of the deployed YARN applications
+1. Supports creation of DNS records for end-points of the deployed YARN applications
2. Record names remain unchanged during restart of containers and/or applications
3. Supports reverse lookups (name based on IP).
4. Supports security using the standards defined by The Domain Name System Security
Extensions (DNSSEC)
5. Highly available
-6. Scalable - The service provides the responsiveness (e.g. low latency) required to
+6. Scalable - The service provides the responsiveness (e.g. low-latency) required to
respond to DNS queries (timeouts yield attempts to invoke other configured name
servers).
-###Deployment properties
+### Deployment properties
1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as
a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a
@@ -55,59 +58,59 @@ DNS standards. The default port for DNS protocols is in a restricted, administra
range (53), so the port is configurable for deployments in which the service may
not be managed via an administrative account.
-##DNS Record Name Structure
+## DNS Record Name Structure
The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see “Preferred Name Syntax” in RFC 1035):
-* **domain** - the name of the cluster DNS domain. This name is provided as a
+* **domain** - the name of the cluster DNS domain. This name is provided as a
configuration property. In addition, it is this name that is configured at a parent DNS
server as the zone name for the defined yDNS zone (the zone for which the parent DNS
server will forward requests to yDNS). E.g. yarncluster.com
-* **username** - the name of the application deployer. This name is the simple shortname (for
+* **username** - the name of the application deployer. This name is the simple short-name (for
e.g. the primary component of the Kerberos principal) associated with the user launching
-the application. As the username is one of the elements of DNS names, it is expected
-that this also confirms DNS name conventions (RFC 1035 linked above) so special translation is performed for names with special characters like hyphens and spaces.
-* **application name** - the name of the deployed YARN application. This name is inferred
-from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
-names. This obviously mandates certain uniqueness properties on application names.
-* **container id** - the YARN assigned ID to a container (e.g.
+the application. As the username is one of the elements of DNS names, it is expected
+that this also confirms DNS name conventions (RFC 1035 linked above), so special translation is performed for names with special characters like hyphens and spaces.
+* **application name** - the name of the deployed YARN application. This name is inferred
+from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
+names. This obviously mandates certain uniqueness properties on application names.
+* **container id** - the YARN assigned ID to a container (e.g.
container_e3741_1454001598828_01_000004)
-* **component name** - the name assigned to the deployed component (for e.g. a master
+* **component name** - the name assigned to the deployed component (for e.g. a master
component). A component is a distributed element of an application or service that is
launched in a YARN container (e.g. an HBase master). One can imagine multiple
-components within an application. A component name is not yet a first class concept in
+components within an application. A component name is not yet a first class concept in
YARN, but is a very useful one that we are introducing here for the sake of yDNS
-entries. Many frameworks like MapReduce, Slider already have component names
-(though, as mentioned, they are not yet supported in YARN in a first class fashion).
-* **api** - the api designation for the exposed endpoint
+entries. Many frameworks like MapReduce, Slider already have component names
+(though, as mentioned, they are not yet supported in YARN in a first class fashion).
+* **api** - the api designation for the exposed endpoint
-###Notes about DNS Names
+### Notes about DNS Names
* In most instances, the DNS names can be easily distinguished by the number of
elements/labels that compose the name. The cluster’s domain name is always the last
element. After that element is parsed out, reading from right to left, the first element
-maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
-“container” or suffix such as “api”. For example, an endpoint published as a
-management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
-* Unique application name (per user) is not currently supported/guaranteed by YARN, but
+maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
+“container” or suffix such as “api”. For example, an endpoint published as a
+management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
+* Unique application name (per user) is not currently supported/guaranteed by YARN, but
it is supported by frameworks such as Apache Slider. The yDNS service currently
leverages the last element of the ZK path entry for the application as an
-application name. These application names have to be unique for a given user.
+application name. These application names have to be unique for a given user.
-##DNS Server Functionality
+## DNS Server Functionality
The primary functions of the DNS service are illustrated in the following diagram:
-###DNS record creation
+### DNS record creation
The following figure illustrates at slightly greater detail the DNS record creation and registration sequence (NOTE: service record updates would follow a similar sequence of steps,
distinguished only by the different event type):
-###DNS record removal
+### DNS record removal
Similarly, record removal follows a similar sequence
@@ -115,7 +118,7 @@ Similarly, record removal follows a similar sequence
(NOTE: The DNS Zone requires a record as an argument for the deletion method, thus
requiring similar parsing logic to identify the specific records that should be removed).
-###DNS Service initialization
+### DNS Service initialization
* The DNS service initializes both UDP and TCP listeners on a configured port. As
noted above, the default port of 53 is in a restricted range that is only accessible to an
account with administrative privileges.
@@ -124,18 +127,18 @@ standard DNS requests from users or other DNS servers (for example, DNS servers
YARN DNS service configured as a forwarder).
## Configuration
-The YARN DNS server reads its configuration properties from the yarnsite.xml file. The following are the DNS associated configuration properties:
+The YARN DNS server reads its configuration properties from the yarn-site.xml file. The following are the DNS associated configuration properties:
| Name | Description |
| ------------ | ------------- |
| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
-| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
-| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
-| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 53. However, since that port falls in a administratoronly range, typical deployments may need to specify an alternate port. |
+| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
+| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
+| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 53. However, since that port falls in a administrator-only range, typical deployments may need to specify an alternate port. |
| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
-| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
-| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
-| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
-| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
-| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
-| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
+| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
+| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
+| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
+| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
+| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
+| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
new file mode 100644
index 0000000..cb7131e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -0,0 +1,494 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.ContainerFailureTracker;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.ServiceMetrics;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.Probe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.START;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP;
+import static org.apache.hadoop.yarn.service.component.ComponentState.*;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
+
+public class Component implements EventHandler<ComponentEvent> {
+ private static final Logger LOG = LoggerFactory.getLogger(Component.class);
+
+ private org.apache.hadoop.yarn.service.api.records.Component componentSpec;
+ private long allocateId;
+ private Priority priority;
+ private ServiceMetrics componentMetrics;
+ private ServiceScheduler scheduler;
+ private ServiceContext context;
+ private AMRMClientAsync<ContainerRequest> amrmClient;
+ private AtomicLong instanceIdCounter = new AtomicLong();
+ private Map<ComponentInstanceId, ComponentInstance> compInstances =
+ new ConcurrentHashMap<>();
+ // component instances to be assigned with a container
+ private List<ComponentInstance> pendingInstances = new LinkedList<>();
+ private ContainerFailureTracker failureTracker;
+ private Probe probe;
+ private final ReentrantReadWriteLock.ReadLock readLock;
+ private final ReentrantReadWriteLock.WriteLock writeLock;
+ public int maxContainerFailurePerComp;
+ // The number of containers failed since last reset. This excludes preempted,
+ // disk_failed containers etc. This will be reset to 0 periodically.
+ public AtomicInteger currentContainerFailure = new AtomicInteger(0);
+
+ private StateMachine<ComponentState, ComponentEventType, ComponentEvent>
+ stateMachine;
+ private AsyncDispatcher compInstanceDispatcher;
+ private static final StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>
+ stateMachineFactory =
+ new StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>(
+ INIT)
+ // INIT will only got to FLEXING
+ .addTransition(INIT, EnumSet.of(STABLE, FLEXING),
+ FLEX, new FlexComponentTransition())
+
+ // container allocated by RM
+ .addTransition(FLEXING, FLEXING, CONTAINER_ALLOCATED,
+ new ContainerAllocatedTransition())
+ // container launched on NM
+ .addTransition(FLEXING, EnumSet.of(STABLE, FLEXING),
+ CONTAINER_STARTED, new ContainerStartedTransition())
+ // container failed while flexing
+ .addTransition(FLEXING, FLEXING, CONTAINER_COMPLETED,
+ new ContainerCompletedTransition())
+ // Flex while previous flex is still in progress
+ .addTransition(FLEXING, EnumSet.of(FLEXING), FLEX,
+ new FlexComponentTransition())
+
+ // container failed while stable
+ .addTransition(STABLE, FLEXING, CONTAINER_COMPLETED,
+ new ContainerCompletedTransition())
+ // Ignore surplus container
+ .addTransition(STABLE, STABLE, CONTAINER_ALLOCATED,
+ new ContainerAllocatedTransition())
+ // Flex by user
+ // For flex up, go to FLEXING state
+ // For flex down, go to STABLE state
+ .addTransition(STABLE, EnumSet.of(STABLE, FLEXING),
+ FLEX, new FlexComponentTransition())
+ .installTopology();
+
+ public Component(
+ org.apache.hadoop.yarn.service.api.records.Component component,
+ long allocateId, ServiceContext context) {
+ this.allocateId = allocateId;
+ this.priority = Priority.newInstance((int) allocateId);
+ this.componentSpec = component;
+ componentMetrics = ServiceMetrics.register(component.getName(),
+ "Metrics for component " + component.getName());
+ componentMetrics
+ .tag("type", "Metrics type [component or service]", "component");
+ this.scheduler = context.scheduler;
+ this.context = context;
+ amrmClient = scheduler.getAmRMClient();
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ this.readLock = lock.readLock();
+ this.writeLock = lock.writeLock();
+ this.stateMachine = stateMachineFactory.make(this);
+ compInstanceDispatcher = scheduler.getCompInstanceDispatcher();
+ failureTracker =
+ new ContainerFailureTracker(context, this);
+ probe = MonitorUtils.getProbe(componentSpec.getReadinessCheck());
+ maxContainerFailurePerComp = componentSpec.getConfiguration()
+ .getPropertyInt(CONTAINER_FAILURE_THRESHOLD, 10);
+ createNumCompInstances(component.getNumberOfContainers());
+ }
+
+ private void createNumCompInstances(long count) {
+ for (int i = 0; i < count; i++) {
+ createOneCompInstance();
+ }
+ }
+
+ private void createOneCompInstance() {
+ ComponentInstanceId id =
+ new ComponentInstanceId(instanceIdCounter.getAndIncrement(),
+ componentSpec.getName());
+ ComponentInstance instance = new ComponentInstance(this, id);
+ compInstances.put(id, instance);
+ pendingInstances.add(instance);
+ }
+
+ private static class FlexComponentTransition implements
+ MultipleArcTransition<Component, ComponentEvent, ComponentState> {
+ // For flex up, go to FLEXING state
+ // For flex down, go to STABLE state
+ @Override
+ public ComponentState transition(Component component,
+ ComponentEvent event) {
+ component.setDesiredContainers((int)event.getDesired());
+ if (!component.areDependenciesReady()) {
+ LOG.info("[FLEX COMPONENT {}]: Flex deferred because dependencies not"
+ + " satisfied.", component.getName());
+ return component.getState();
+ }
+ if (component.getState() == INIT) {
+ // This happens on init
+ LOG.info("[INIT COMPONENT " + component.getName() + "]: " + event
+ .getDesired() + " instances.");
+ component.requestContainers(event.getDesired());
+ return FLEXING;
+ }
+ long before = component.getComponentSpec().getNumberOfContainers();
+ long delta = event.getDesired() - before;
+ component.getComponentSpec().setNumberOfContainers(event.getDesired());
+ if (delta > 0) {
+ // Scale up
+ LOG.info("[FLEX UP COMPONENT " + component.getName() + "]: scaling up from "
+ + before + " to " + event.getDesired());
+ component.requestContainers(delta);
+ component.createNumCompInstances(delta);
+ return FLEXING;
+ } else if (delta < 0){
+ delta = 0 - delta;
+ // scale down
+ LOG.info("[FLEX DOWN COMPONENT " + component.getName()
+ + "]: scaling down from " + before + " to " + event.getDesired());
+ List<ComponentInstance> list =
+ new ArrayList<>(component.compInstances.values());
+
+ // sort in Most recent -> oldest order, destroy most recent ones.
+ Collections.sort(list, Collections.reverseOrder());
+ for (int i = 0; i < delta; i++) {
+ ComponentInstance instance = list.get(i);
+ // remove the instance
+ component.compInstances.remove(instance.getCompInstanceId());
+ component.pendingInstances.remove(instance);
+ component.componentMetrics.containersFailed.incr();
+ component.componentMetrics.containersRunning.decr();
+ // decrement id counter
+ component.instanceIdCounter.decrementAndGet();
+ instance.destroy();
+ }
+ return STABLE;
+ } else {
+ LOG.info("[FLEX COMPONENT " + component.getName() + "]: already has " +
+ event.getDesired() + " instances, ignoring");
+ return STABLE;
+ }
+ }
+ }
+
+ private static class ContainerAllocatedTransition extends BaseTransition {
+ @Override
+ public void transition(Component component, ComponentEvent event) {
+ component.assignContainerToCompInstance(event.getContainer());
+ }
+ }
+
+ private static class ContainerStartedTransition implements
+ MultipleArcTransition<Component,ComponentEvent,ComponentState> {
+
+ @Override public ComponentState transition(Component component,
+ ComponentEvent event) {
+ component.compInstanceDispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(event.getInstance().getContainerId(),
+ START));
+ component.incRunningContainers();
+ return checkIfStable(component);
+ }
+ }
+
+ private static ComponentState checkIfStable(Component component) {
+ // if desired == running
+ if (component.componentMetrics.containersRunning.value() == component
+ .getComponentSpec().getNumberOfContainers()) {
+ return STABLE;
+ } else {
+ return FLEXING;
+ }
+ }
+
+ private static class ContainerCompletedTransition extends BaseTransition {
+ @Override
+ public void transition(Component component, ComponentEvent event) {
+ component.updateMetrics(event.getStatus());
+
+ // add back to pending list
+ component.pendingInstances.add(event.getInstance());
+ LOG.info(
+ "[COMPONENT {}]: {} completed, num pending comp instances increased to {}.",
+ component.getName(), event.getStatus().getContainerId(),
+ component.pendingInstances.size());
+ component.compInstanceDispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(event.getStatus().getContainerId(),
+ STOP).setStatus(event.getStatus()));
+ }
+ }
+
+ public ServiceMetrics getCompMetrics () {
+ return componentMetrics;
+ }
+
+ private void assignContainerToCompInstance(Container container) {
+ if (pendingInstances.size() == 0) {
+ LOG.info(
+ "[COMPONENT {}]: No pending component instance left, release surplus container {}",
+ getName(), container.getId());
+ scheduler.getAmRMClient().releaseAssignedContainer(container.getId());
+ componentMetrics.surplusContainers.incr();
+ scheduler.getServiceMetrics().surplusContainers.incr();
+ return;
+ }
+ ComponentInstance instance = pendingInstances.remove(0);
+ LOG.info(
+ "[COMPONENT {}]: {} allocated, num pending component instances reduced to {}",
+ getName(), container.getId(), pendingInstances.size());
+ instance.setContainer(container);
+ scheduler.addLiveCompInstance(container.getId(), instance);
+ LOG.info(
+ "[COMPONENT {}]: Assigned {} to component instance {} and launch on host {} ",
+ getName(), container.getId(), instance.getCompInstanceName(),
+ container.getNodeId());
+ scheduler.getContainerLaunchService()
+ .launchCompInstance(scheduler.getApp(), instance, container);
+ }
+
+ @SuppressWarnings({ "unchecked" })
+ public void requestContainers(long count) {
+ Resource resource = Resource
+ .newInstance(componentSpec.getResource().getMemoryMB(),
+ componentSpec.getResource().getCpus());
+
+ for (int i = 0; i < count; i++) {
+ //TODO Once YARN-5468 is done, use that for anti-affinity
+ ContainerRequest request =
+ ContainerRequest.newBuilder().capability(resource).priority(priority)
+ .allocationRequestId(allocateId).relaxLocality(true).build();
+ amrmClient.addContainerRequest(request);
+ }
+ }
+
+ private void setDesiredContainers(int n) {
+ int delta = n - scheduler.getServiceMetrics().containersDesired.value();
+ if (delta > 0) {
+ scheduler.getServiceMetrics().containersDesired.incr(delta);
+ } else {
+ scheduler.getServiceMetrics().containersDesired.decr(delta);
+ }
+ componentMetrics.containersDesired.set(n);
+ }
+
+
+
+ private void updateMetrics(ContainerStatus status) {
+ switch (status.getExitStatus()) {
+ case SUCCESS:
+ componentMetrics.containersSucceeded.incr();
+ scheduler.getServiceMetrics().containersSucceeded.incr();
+ return;
+ case PREEMPTED:
+ componentMetrics.containersPreempted.incr();
+ scheduler.getServiceMetrics().containersPreempted.incr();
+ break;
+ case DISKS_FAILED:
+ componentMetrics.containersDiskFailure.incr();
+ scheduler.getServiceMetrics().containersDiskFailure.incr();
+ break;
+ default:
+ break;
+ }
+
+ // containersFailed include preempted, disks_failed etc.
+ componentMetrics.containersFailed.incr();
+ scheduler.getServiceMetrics().containersFailed.incr();
+
+ // dec running container
+ decRunningContainers();
+
+ if (Apps.shouldCountTowardsNodeBlacklisting(status.getExitStatus())) {
+ String host = scheduler.getLiveInstances().get(status.getContainerId())
+ .getNodeId().getHost();
+ failureTracker.incNodeFailure(host);
+ currentContainerFailure.getAndIncrement() ;
+ }
+ }
+
+ public boolean areDependenciesReady() {
+ List<String> dependencies = componentSpec.getDependencies();
+ if (SliderUtils.isEmpty(dependencies)) {
+ return true;
+ }
+ for (String dependency : dependencies) {
+ Component dependentComponent =
+ scheduler.getAllComponents().get(dependency);
+ if (dependentComponent == null) {
+ LOG.error("Couldn't find dependency {} for {} (should never happen)",
+ dependency, getName());
+ continue;
+ }
+ if (dependentComponent.getNumReadyInstances() < dependentComponent
+ .getNumDesiredInstances()) {
+ LOG.info("[COMPONENT {}]: Dependency {} not satisfied, only {} of {}"
+ + " instances are ready.", getName(), dependency,
+ dependentComponent.getNumReadyInstances(),
+ dependentComponent.getNumDesiredInstances());
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void incRunningContainers() {
+ componentMetrics.containersRunning.incr();
+ scheduler.getServiceMetrics().containersRunning.incr();
+ }
+
+ public void incContainersReady() {
+ componentMetrics.containersReady.incr();
+ }
+
+ public void decContainersReady() {
+ componentMetrics.containersReady.decr();
+ }
+
+ private void decRunningContainers() {
+ componentMetrics.containersRunning.decr();
+ scheduler.getServiceMetrics().containersRunning.decr();
+ }
+
+ public int getNumReadyInstances() {
+ return componentMetrics.containersReady.value();
+ }
+
+ public int getNumRunningInstances() {
+ return componentMetrics.containersRunning.value();
+ }
+
+ public int getNumDesiredInstances() {
+ return componentMetrics.containersDesired.value();
+ }
+
+ public Map<ComponentInstanceId, ComponentInstance> getAllComponentInstances() {
+ return compInstances;
+ }
+
+ public org.apache.hadoop.yarn.service.api.records.Component getComponentSpec() {
+ return this.componentSpec;
+ }
+
+ public void resetCompFailureCount() {
+ LOG.info("[COMPONENT {}]: Reset container failure count from {} to 0.",
+ getName(), currentContainerFailure.get());
+ currentContainerFailure.set(0);
+ failureTracker.resetContainerFailures();
+ }
+
+ public Probe getProbe() {
+ return probe;
+ }
+
+ public Priority getPriority() {
+ return priority;
+ }
+
+ public long getAllocateId() {
+ return allocateId;
+ }
+
+ public String getName () {
+ return componentSpec.getName();
+ }
+
+ public ComponentState getState() {
+ this.readLock.lock();
+
+ try {
+ return this.stateMachine.getCurrentState();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+ public ServiceScheduler getScheduler() {
+ return scheduler;
+ }
+
+ @Override
+ public void handle(ComponentEvent event) {
+ try {
+ writeLock.lock();
+ ComponentState oldState = getState();
+ try {
+ stateMachine.doTransition(event.getType(), event);
+ } catch (InvalidStateTransitionException e) {
+ LOG.error(MessageFormat.format("[COMPONENT {0}]: Invalid event {1} at {2}",
+ componentSpec.getName(), event.getType(), oldState), e);
+ }
+ if (oldState != getState()) {
+ LOG.info("[COMPONENT {}] Transitioned from {} to {} on {} event.",
+ componentSpec.getName(), oldState, getState(), event.getType());
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private static class BaseTransition implements
+ SingleArcTransition<Component, ComponentEvent> {
+
+ @Override public void transition(Component component,
+ ComponentEvent event) {
+ }
+ }
+
+ public ServiceContext getContext() {
+ return context;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
new file mode 100644
index 0000000..d93dcf1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+public class ComponentEvent extends AbstractEvent<ComponentEventType> {
+ private long desired;
+ private final String name;
+ private final ComponentEventType type;
+ private Container container;
+ private ComponentInstance instance;
+ private ContainerStatus status;
+
+ public ComponentEvent(String name, ComponentEventType type) {
+ super(type);
+ this.name = name;
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public ComponentEventType getType() {
+ return type;
+ }
+
+ public long getDesired() {
+ return desired;
+ }
+
+ public ComponentEvent setDesired(long desired) {
+ this.desired = desired;
+ return this;
+ }
+
+ public Container getContainer() {
+ return container;
+ }
+
+ public ComponentEvent setContainer(Container container) {
+ this.container = container;
+ return this;
+ }
+
+ public ComponentInstance getInstance() {
+ return instance;
+ }
+
+ public ComponentEvent setInstance(ComponentInstance instance) {
+ this.instance = instance;
+ return this;
+ }
+
+ public ContainerStatus getStatus() {
+ return status;
+ }
+
+ public ComponentEvent setStatus(ContainerStatus status) {
+ this.status = status;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
new file mode 100644
index 0000000..6729699
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+public enum ComponentEventType {
+ FLEX,
+ CONTAINER_ALLOCATED,
+ CONTAINER_STARTED,
+ CONTAINER_COMPLETED
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
new file mode 100644
index 0000000..a5f9ff4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+public enum ComponentState {
+ INIT,
+ FLEXING,
+ STABLE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
new file mode 100644
index 0000000..7d6525b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -0,0 +1,493 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.client.api.NMClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.BoundedAppender;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.Date;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
+import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState.*;
+
+public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
+ Comparable<ComponentInstance> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ComponentInstance.class);
+
+ private StateMachine<ComponentInstanceState, ComponentInstanceEventType,
+ ComponentInstanceEvent> stateMachine;
+ private Component component;
+ private final ReadLock readLock;
+ private final WriteLock writeLock;
+
+ private ComponentInstanceId compInstanceId = null;
+ private Path compInstanceDir;
+ private Container container;
+ private YarnRegistryViewForProviders yarnRegistryOperations;
+ private FileSystem fs;
+ private boolean timelineServiceEnabled = false;
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+ private ServiceScheduler scheduler;
+ private BoundedAppender diagnostics = new BoundedAppender(64 * 1024);
+ private volatile ScheduledFuture containerStatusFuture;
+ private volatile ContainerStatus status;
+ private long containerStartedTime = 0;
+ // This container object is used for rest API query
+ private org.apache.hadoop.yarn.service.api.records.Container containerSpec;
+
+ private static final StateMachineFactory<ComponentInstance,
+ ComponentInstanceState, ComponentInstanceEventType, ComponentInstanceEvent>
+ stateMachineFactory =
+ new StateMachineFactory<ComponentInstance, ComponentInstanceState,
+ ComponentInstanceEventType, ComponentInstanceEvent>(INIT)
+ .addTransition(INIT, STARTED, START,
+ new ContainerStartedTransition())
+
+ //From Running
+ .addTransition(STARTED, INIT, STOP,
+ new ContainerStoppedTransition())
+ .addTransition(STARTED, READY, BECOME_READY,
+ new ContainerBecomeReadyTransition())
+
+ // FROM READY
+ .addTransition(READY, STARTED, BECOME_NOT_READY,
+ new ContainerBecomeNotReadyTransition())
+ .addTransition(READY, INIT, STOP, new ContainerStoppedTransition())
+ .installTopology();
+
+
+
+ public ComponentInstance(Component component,
+ ComponentInstanceId compInstanceId) {
+ this.stateMachine = stateMachineFactory.make(this);
+ this.component = component;
+ this.compInstanceId = compInstanceId;
+ this.scheduler = component.getScheduler();
+ this.yarnRegistryOperations =
+ component.getScheduler().getYarnRegistryOperations();
+ this.serviceTimelinePublisher =
+ component.getScheduler().getServiceTimelinePublisher();
+ if (YarnConfiguration
+ .timelineServiceV2Enabled(component.getScheduler().getConfig())) {
+ this.timelineServiceEnabled = true;
+ }
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ this.readLock = lock.readLock();
+ this.writeLock = lock.writeLock();
+ this.fs = scheduler.getContext().fs.getFileSystem();
+ }
+
+ private static class ContainerStartedTransition extends BaseTransition {
+ @Override public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ // Query container status for ip and host
+ compInstance.containerStatusFuture =
+ compInstance.scheduler.executorService.scheduleAtFixedRate(
+ new ContainerStatusRetriever(compInstance.scheduler,
+ compInstance.getContainerId(), compInstance), 0, 1,
+ TimeUnit.SECONDS);
+
+ org.apache.hadoop.yarn.service.api.records.Container container =
+ new org.apache.hadoop.yarn.service.api.records.Container();
+ container.setId(compInstance.getContainerId().toString());
+ container.setLaunchTime(new Date());
+ container.setState(ContainerState.RUNNING_BUT_UNREADY);
+ container.setBareHost(compInstance.container.getNodeId().getHost());
+ container.setComponentName(compInstance.getCompInstanceName());
+ if (compInstance.containerSpec != null) {
+ // remove the previous container.
+ compInstance.getCompSpec().removeContainer(compInstance.containerSpec);
+ }
+ compInstance.containerSpec = container;
+ compInstance.getCompSpec().addContainer(container);
+ compInstance.containerStartedTime = System.currentTimeMillis();
+
+ if (compInstance.timelineServiceEnabled) {
+ compInstance.serviceTimelinePublisher
+ .componentInstanceStarted(container, compInstance);
+ }
+ }
+ }
+
+ private static class ContainerBecomeReadyTransition extends BaseTransition {
+ @Override
+ public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ compInstance.component.incContainersReady();
+ compInstance.containerSpec.setState(ContainerState.READY);
+ }
+ }
+
+ private static class ContainerBecomeNotReadyTransition extends BaseTransition {
+ @Override
+ public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ compInstance.component.decContainersReady();
+ compInstance.containerSpec.setState(ContainerState.RUNNING_BUT_UNREADY);
+ }
+ }
+
+ private static class ContainerStoppedTransition extends BaseTransition {
+ @Override
+ public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ // re-ask the failed container.
+ Component comp = compInstance.component;
+ comp.requestContainers(1);
+ LOG.info(compInstance.getCompInstanceId()
+ + ": Container completed. Requested a new container." + System
+ .lineSeparator() + " exitStatus={}, diagnostics={}.",
+ event.getStatus().getExitStatus(),
+ event.getStatus().getDiagnostics());
+ String containerDiag =
+ compInstance.getCompInstanceId() + ": " + event.getStatus()
+ .getDiagnostics();
+ compInstance.diagnostics.append(containerDiag + System.lineSeparator());
+
+ boolean shouldExit = false;
+ // check if it exceeds the failure threshold
+ if (comp.currentContainerFailure.get() > comp.maxContainerFailurePerComp) {
+ String exitDiag = MessageFormat.format(
+ "[COMPONENT {0}]: Failed {1} times, exceeded the limit - {2}. Shutting down now... "
+ + System.lineSeparator(),
+ comp.getName(), comp.currentContainerFailure.get(), comp.maxContainerFailurePerComp);
+ compInstance.diagnostics.append(exitDiag);
+ // append to global diagnostics that will be reported to RM.
+ comp.getScheduler().getDiagnostics().append(containerDiag);
+ comp.getScheduler().getDiagnostics().append(exitDiag);
+ LOG.warn(exitDiag);
+ shouldExit = true;
+ }
+
+ // clean up registry
+ // hdfs dir content will be overwritten when a new container gets started,
+ // so no need remove.
+ compInstance.scheduler.executorService
+ .submit(compInstance::cleanupRegistry);
+
+ // remove the failed ContainerId -> CompInstance mapping
+ comp.getScheduler().removeLiveCompInstance(event.getContainerId());
+
+ if (compInstance.timelineServiceEnabled) {
+ // record in ATS
+ compInstance.serviceTimelinePublisher
+ .componentInstanceFinished(compInstance,
+ event.getStatus().getExitStatus(), event.getStatus().getState(),
+ containerDiag);
+ }
+
+ compInstance.containerSpec.setState(ContainerState.STOPPED);
+ if (shouldExit) {
+ // Sleep for 5 seconds in hope that the state can be recorded in ATS.
+ // in case there's a client polling the comp state, it can be notified.
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted on sleep while exiting.", e);
+ }
+ ExitUtil.terminate(-1);
+ }
+ }
+ }
+
+ public ComponentInstanceState getState() {
+ this.readLock.lock();
+
+ try {
+ return this.stateMachine.getCurrentState();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public void handle(ComponentInstanceEvent event) {
+ try {
+ writeLock.lock();
+ ComponentInstanceState oldState = getState();
+ try {
+ stateMachine.doTransition(event.getType(), event);
+ } catch (InvalidStateTransitionException e) {
+ LOG.error(getCompInstanceId() + ": Invalid event " + event.getType() +
+ " at " + oldState, e);
+ }
+ if (oldState != getState()) {
+ LOG.info(getCompInstanceId() + " Transitioned from " + oldState + " to "
+ + getState() + " on " + event.getType() + " event");
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public void setContainer(Container container) {
+ this.container = container;
+ this.compInstanceId.setContainerId(container.getId());
+ }
+
+ public String getCompInstanceName() {
+ return compInstanceId.getCompInstanceName();
+ }
+
+ public ContainerStatus getContainerStatus() {
+ return status;
+ }
+
+ public void updateContainerStatus(ContainerStatus status) {
+ this.status = status;
+ org.apache.hadoop.yarn.service.api.records.Container container =
+ getCompSpec().getContainer(getContainerId().toString());
+ if (container != null) {
+ container.setIp(StringUtils.join(",", status.getIPs()));
+ container.setHostname(status.getHost());
+ if (timelineServiceEnabled) {
+ serviceTimelinePublisher.componentInstanceUpdated(container);
+ }
+ }
+ updateServiceRecord(yarnRegistryOperations, status);
+ }
+
+ public ContainerId getContainerId() {
+ return container.getId();
+ }
+
+ public String getCompName() {
+ return compInstanceId.getCompName();
+ }
+
+ public void setCompInstanceDir(Path dir) {
+ this.compInstanceDir = dir;
+ }
+
+ public Component getComponent() {
+ return component;
+ }
+
+ public Container getContainer() {
+ return container;
+ }
+
+ public ComponentInstanceId getCompInstanceId() {
+ return compInstanceId;
+ }
+
+ public NodeId getNodeId() {
+ return this.container.getNodeId();
+ }
+
+ public org.apache.hadoop.yarn.service.api.records.Component getCompSpec() {
+ return component.getComponentSpec();
+ }
+
+ private static class BaseTransition implements
+ SingleArcTransition<ComponentInstance, ComponentInstanceEvent> {
+
+ @Override public void transition(ComponentInstance compInstance,
+ ComponentInstanceEvent event) {
+ }
+ }
+
+ public ProbeStatus ping() {
+ if (component.getProbe() == null) {
+ ProbeStatus status = new ProbeStatus();
+ status.setSuccess(true);
+ return status;
+ }
+ return component.getProbe().ping(this);
+ }
+
+ // Write service record into registry
+ private void updateServiceRecord(
+ YarnRegistryViewForProviders yarnRegistry, ContainerStatus status) {
+ ServiceRecord record = new ServiceRecord();
+ String containerId = status.getContainerId().toString();
+ record.set(YarnRegistryAttributes.YARN_ID, containerId);
+ record.description = getCompInstanceName();
+ record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.CONTAINER);
+ record.set("yarn:ip", status.getIPs());
+ record.set("yarn:hostname", status.getHost());
+ try {
+ yarnRegistry
+ .putComponent(RegistryPathUtils.encodeYarnID(containerId), record);
+ } catch (IOException e) {
+ LOG.error(
+ "Failed to update service record in registry: " + containerId + "");
+ }
+ }
+
+ // Release the container , cleanup registry, hdfs dir, and record in ATS
+ public void destroy() {
+ LOG.info(getCompInstanceId() + ": Flexed down by user, destroying.");
+ diagnostics.append(getCompInstanceId() + ": Flexed down by user");
+ if (container != null) {
+ scheduler.removeLiveCompInstance(container.getId());
+ component.getScheduler().getAmRMClient()
+ .releaseAssignedContainer(container.getId());
+ getCompSpec().removeContainer(containerSpec);
+ }
+ if (timelineServiceEnabled) {
+ serviceTimelinePublisher
+ .componentInstanceFinished(this, KILLED_BY_APPMASTER, COMPLETE,
+ diagnostics.toString());
+ }
+ scheduler.executorService.submit(this::cleanupRegistryAndCompHdfsDir);
+ }
+
+ private void cleanupRegistry() {
+ ContainerId containerId = getContainerId();
+ String cid = RegistryPathUtils.encodeYarnID(containerId.toString());
+ try {
+ yarnRegistryOperations.deleteComponent(getCompInstanceId(), cid);
+ } catch (IOException e) {
+ LOG.error(getCompInstanceId() + ": Failed to delete registry", e);
+ }
+ }
+
+ //TODO Maybe have a dedicated cleanup service.
+ public void cleanupRegistryAndCompHdfsDir() {
+ cleanupRegistry();
+ try {
+ if (compInstanceDir != null && fs.exists(compInstanceDir)) {
+ boolean deleted = fs.delete(compInstanceDir, true);
+ if (!deleted) {
+ LOG.error(getCompInstanceId()
+ + ": Failed to delete component instance dir: "
+ + compInstanceDir);
+ } else {
+ LOG.info(getCompInstanceId() + ": Deleted component instance dir: "
+ + compInstanceDir);
+ }
+ }
+ } catch (IOException e) {
+ LOG.warn(getCompInstanceId() + ": Failed to delete directory", e);
+ }
+ }
+
+ // Query container status until ip and hostname are available and update
+ // the service record into registry service
+ private static class ContainerStatusRetriever implements Runnable {
+ private ContainerId containerId;
+ private NodeId nodeId;
+ private NMClient nmClient;
+ private ComponentInstance instance;
+ ContainerStatusRetriever(ServiceScheduler scheduler,
+ ContainerId containerId, ComponentInstance instance) {
+ this.containerId = containerId;
+ this.nodeId = instance.getNodeId();
+ this.nmClient = scheduler.getNmClient().getClient();
+ this.instance = instance;
+ }
+ @Override public void run() {
+ ContainerStatus status = null;
+ try {
+ status = nmClient.getContainerStatus(containerId, nodeId);
+ } catch (Exception e) {
+ if (e instanceof YarnException) {
+ throw new YarnRuntimeException(
+ instance.compInstanceId + " Failed to get container status on "
+ + nodeId + " , cancelling.", e);
+ }
+ LOG.error(instance.compInstanceId + " Failed to get container status on "
+ + nodeId + ", will try again", e);
+ return;
+ }
+ if (SliderUtils.isEmpty(status.getIPs()) || SliderUtils
+ .isUnset(status.getHost())) {
+ return;
+ }
+ instance.updateContainerStatus(status);
+ LOG.info(
+ instance.compInstanceId + " IP = " + status.getIPs() + ", host = "
+ + status.getHost() + ", cancel container status retriever");
+ instance.containerStatusFuture.cancel(false);
+ }
+ }
+
+ @Override
+ public int compareTo(ComponentInstance to) {
+ long delta = containerStartedTime - to.containerStartedTime;
+ if (delta == 0) {
+ return getCompInstanceId().compareTo(to.getCompInstanceId());
+ } else if (delta < 0) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ComponentInstance instance = (ComponentInstance) o;
+
+ if (containerStartedTime != instance.containerStartedTime)
+ return false;
+ return compInstanceId.equals(instance.compInstanceId);
+ }
+
+ @Override public int hashCode() {
+ int result = compInstanceId.hashCode();
+ result = 31 * result + (int) (containerStartedTime ^ (containerStartedTime
+ >>> 32));
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
new file mode 100644
index 0000000..707b034
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class ComponentInstanceEvent
+ extends AbstractEvent<ComponentInstanceEventType> {
+
+ private ContainerId id;
+ private ContainerStatus status;
+ private boolean shouldDestroy = false;
+
+ public ComponentInstanceEvent(ContainerId containerId,
+ ComponentInstanceEventType componentInstanceEventType) {
+ super(componentInstanceEventType);
+ this.id = containerId;
+ }
+
+ public ContainerId getContainerId() {
+ return id;
+ }
+
+ public ContainerStatus getStatus() {
+ return this.status;
+ }
+
+ public ComponentInstanceEvent setStatus(ContainerStatus status) {
+ this.status = status;
+ return this;
+ }
+
+ public void setShouldDestroy() {
+ shouldDestroy = true;
+ }
+
+ public boolean shouldDestroy() {
+ return shouldDestroy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
new file mode 100644
index 0000000..1a880ba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+public enum ComponentInstanceEventType {
+ START,
+ STOP,
+ BECOME_READY,
+ BECOME_NOT_READY
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceId.java
new file mode 100644
index 0000000..14387ba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceId.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+public class ComponentInstanceId implements Comparable<ComponentInstanceId> {
+
+ private long Id;
+ private String name;
+ private ContainerId containerId;
+
+ public ComponentInstanceId(long id, String name) {
+ Id = id;
+ this.name = name;
+ }
+
+ public long getId() {
+ return Id;
+ }
+
+ public String getCompName() {
+ return name;
+ }
+
+ public String getCompInstanceName() {
+ return getCompName() + "-" + getId();
+ }
+
+ public void setContainerId(ContainerId containerId) {
+ this.containerId = containerId;
+ }
+
+ @Override
+ public String toString() {
+ if (containerId == null) {
+ return "[COMPINSTANCE " + getCompInstanceName() + "]";
+ } else {
+ return "[COMPINSTANCE " + getCompInstanceName() + " : " + containerId + "]";
+ }
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ComponentInstanceId that = (ComponentInstanceId) o;
+
+ if (getId() != that.getId())
+ return false;
+ return getCompName() != null ? getCompName().equals(that.getCompName()) :
+ that.getCompName() == null;
+
+ }
+
+ @Override public int hashCode() {
+ int result = (int) (getId() ^ (getId() >>> 32));
+ result = 31 * result + (getCompName() != null ? getCompName().hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public int compareTo(ComponentInstanceId to) {
+ int delta = this.getCompName().compareTo(to.getCompName());
+ if (delta == 0) {
+ return Long.compare(this.getId(), to.getId());
+ } else if (delta < 0) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
new file mode 100644
index 0000000..f5de5cb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component.instance;
+
+public enum ComponentInstanceState {
+ INIT,
+ STARTED,
+ READY,
+ UPGRADING
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
new file mode 100644
index 0000000..6de2dc0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+public interface RestApiConstants {
+
+ // Rest endpoints
+ String CONTEXT_ROOT = "/ws/v1";
+ String VERSION = "/services/version";
+ String SERVICE_ROOT_PATH = "/services";
+ String SERVICE_PATH = "/services/{service_name}";
+ String COMPONENT_PATH = "/services/{service_name}/components/{component_name}";
+
+ // Query param
+ String SERVICE_NAME = "service_name";
+ String COMPONENT_NAME = "component_name";
+
+ String DEFAULT_COMPONENT_NAME = "default";
+
+ String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
+ String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
+ Long DEFAULT_UNLIMITED_LIFETIME = -1l;
+
+ Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
+ Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
+ Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
+ Integer ERROR_CODE_APP_NAME_INVALID = 404004;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
new file mode 100644
index 0000000..ee270cb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+
+public interface SliderExitCodes extends LauncherExitCodes {
+
+ /**
+ * starting point for exit codes; not an exception itself
+ */
+ int _EXIT_CODE_BASE = 64;
+
+ /**
+ * service entered the failed state: {@value}
+ */
+ int EXIT_YARN_SERVICE_FAILED = 65;
+
+ /**
+ * service was killed: {@value}
+ */
+ int EXIT_YARN_SERVICE_KILLED = 66;
+
+ /**
+ * timeout on monitoring client: {@value}
+ */
+ int EXIT_TIMED_OUT = 67;
+
+ /**
+ * service finished with an error: {@value}
+ */
+ int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
+
+ /**
+ * the service instance is unknown: {@value}
+ */
+ int EXIT_UNKNOWN_INSTANCE = 69;
+
+ /**
+ * the service instance is in the wrong state for that operation: {@value}
+ */
+ int EXIT_BAD_STATE = 70;
+
+ /**
+ * A spawned master process failed
+ */
+ int EXIT_PROCESS_FAILED = 71;
+
+ /**
+ * The instance failed -too many containers were
+ * failing or some other threshold was reached
+ */
+ int EXIT_DEPLOYMENT_FAILED = 72;
+
+ /**
+ * The service is live -and the requested operation
+ * does not work if the cluster is running
+ */
+ int EXIT_APPLICATION_IN_USE = 73;
+
+ /**
+ * There already is an service instance of that name
+ * when an attempt is made to create a new instance
+ */
+ int EXIT_INSTANCE_EXISTS = 75;
+
+ /**
+ * Exit code when the configurations in valid/incomplete: {@value}
+ */
+ int EXIT_BAD_CONFIGURATION = 77;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
new file mode 100644
index 0000000..1968e95
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+
+public class YarnServiceConf {
+
+ // Retry settings for the ServiceClient to talk to Service AppMaster
+ public static final String CLIENT_AM_RETRY_MAX_WAIT_MS = "yarn.service.client-am.retry.max-wait-ms";
+ public static final String CLIENT_AM_RETRY_MAX_INTERVAL_MS = "yarn.service.client-am.retry-interval-ms";
+
+ // Retry settings for container failures
+ public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
+ public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval";
+
+ public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
+ public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
+ public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
+
+ public static final String YARN_QUEUE = "yarn.service.queue";
+
+ public static final String API_SERVER_ADDRESS = "yarn.service.api-server.address";
+ public static final String DEFAULT_API_SERVER_ADDRESS = "0.0.0.0:";
+ public static final int DEFAULT_API_SERVER_PORT = 9191;
+
+ /**
+ * The yarn service base path:
+ * Defaults to HomeDir/.yarn/
+ */
+ public static final String YARN_SERVICE_BASE_PATH = "yarn.service.base.path";
+
+ //TODO rename
+ /** Declare that a keytab must be provided */
+ public static final String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
+ public static final String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ public static final String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
+ public static final String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+
+ /**
+ * maximum number of failed containers (in a single component)
+ * before the app exits
+ */
+ public static final String CONTAINER_FAILURE_THRESHOLD =
+ "yarn.service.container-failure-per-component.threshold";
+ /**
+ * Maximum number of container failures on a node before the node is blacklisted
+ */
+ public static final String NODE_BLACKLIST_THRESHOLD =
+ "yarn.service.node-blacklist.threshold";
+
+ /**
+ * The failure count for CONTAINER_FAILURE_THRESHOLD and NODE_BLACKLIST_THRESHOLD
+ * gets reset periodically, the unit is seconds.
+ */
+ public static final String CONTAINER_FAILURE_WINDOW =
+ "yarn.service.failure-count-reset.window";
+
+ /**
+ * interval between readiness checks.
+ */
+ public static final String READINESS_CHECK_INTERVAL = "yarn.service.readiness-check-interval.seconds";
+ public static final int DEFAULT_READINESS_CHECK_INTERVAL = 30; // seconds
+
+ /**
+ * Get long value for the property. First get from the userConf, if not
+ * present, get from systemConf.
+ *
+ * @param name name of the property
+ * @param defaultValue default value of the property, if it is not defined in
+ * userConf and systemConf.
+ * @param userConf Configuration provided by client in the JSON definition
+ * @param systemConf The YarnConfiguration in the system.
+ * @return long value for the property
+ */
+ public static long getLong(String name, long defaultValue,
+ Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
+ return userConf.getPropertyLong(name, systemConf.getLong(name, defaultValue));
+ }
+
+ public static int getInt(String name, int defaultValue,
+ Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
+ return userConf.getPropertyInt(name, systemConf.getInt(name, defaultValue));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
new file mode 100644
index 0000000..e5ed703
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+public interface YarnServiceConstants {
+
+ /**
+ * The path under which cluster and temp data are stored
+ */
+ String SERVICE_BASE_DIRECTORY = ".yarn";
+
+ /**
+ * The paths under which Service AM dependency libraries are stored
+ */
+ String DEPENDENCY_LOCALIZED_DIR_LINK = "service_dep";
+ String DEPENDENCY_DIR = "/yarn-services/%s/";
+ String DEPENDENCY_TAR_GZ_FILE_NAME = "service-dep";
+ String DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
+ String DEPENDENCY_DIR_PERMISSIONS = "755";
+
+ /**
+ * Service type for YARN service
+ */
+ String APP_TYPE = "yarn-service";
+
+ String KEYTAB_DIR = "keytabs";
+ String RESOURCE_DIR = "resources";
+
+
+ String SERVICES_DIRECTORY = "services";
+
+ /**
+ * JVM property to define the service lib directory;
+ * this is set by the yarn.sh script
+ */
+ String PROPERTY_LIB_DIR = "service.libdir";
+
+ /**
+ * name of generated dir for this conf
+ */
+ String SUBMITTED_CONF_DIR = "conf";
+
+ /**
+ * Service AM log4j file name
+ */
+ String YARN_SERVICE_LOG4J_FILENAME = "yarnservice-log4j.properties";
+
+ /**
+ * Log4j sysprop to name the resource
+ */
+ String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
+
+ /**
+ * sysprop for Service AM log4j directory
+ */
+ String SYSPROP_LOG_DIR = "LOG_DIR";
+
+ String TMP_DIR_PREFIX = "tmp";
+
+
+ String SERVICE_CORE_JAR = "yarn-service-core.jar";
+
+ String STDOUT_AM = "serviceam-out.txt";
+ String STDERR_AM = "serviceam-err.txt";
+
+ String HADOOP_USER_NAME = "HADOOP_USER_NAME";
+
+ String APP_CONF_DIR = "conf";
+
+ String APP_LIB_DIR = "lib";
+
+ String OUT_FILE = "stdout.txt";
+ String ERR_FILE = "stderr.txt";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
new file mode 100644
index 0000000..e4eae20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
+
+/**
+ * Launcher of applications: base class
+ */
+public class AbstractLauncher {
+ private static final Logger log =
+ LoggerFactory.getLogger(AbstractLauncher.class);
+ public static final String CLASSPATH = "CLASSPATH";
+ /**
+ * Filesystem to use for the launch
+ */
+ protected final CoreFileSystem coreFileSystem;
+ /**
+ * Env vars; set up at final launch stage
+ */
+ protected final Map<String, String> envVars = new HashMap<>();
+ protected final ContainerLaunchContext containerLaunchContext =
+ Records.newRecord(ContainerLaunchContext.class);
+ protected final List<String> commands = new ArrayList<>(20);
+ protected final Map<String, LocalResource> localResources = new HashMap<>();
+ protected final Map<String, String> mountPaths = new HashMap<>();
+ private final Map<String, ByteBuffer> serviceData = new HashMap<>();
+ // security
+ protected final Credentials credentials;
+ protected boolean yarnDockerMode = false;
+ protected String dockerImage;
+ protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
+ protected String dockerHostname;
+ protected String runPrivilegedContainer;
+
+
+ /**
+ * Create instance.
+ * @param coreFileSystem filesystem
+ * @param credentials initial set of credentials -null is permitted
+ */
+ public AbstractLauncher(
+ CoreFileSystem coreFileSystem,
+ Credentials credentials) {
+ this.coreFileSystem = coreFileSystem;
+ this.credentials = credentials != null ? credentials: new Credentials();
+ }
+
+ public void setYarnDockerMode(boolean yarnDockerMode){
+ this.yarnDockerMode = yarnDockerMode;
+ }
+
+ /**
+ * Get the env vars to work on
+ * @return env vars
+ */
+ public Map<String, String> getEnv() {
+ return envVars;
+ }
+
+ /**
+ * Get the launch commands.
+ * @return the live list of commands
+ */
+ public List<String> getCommands() {
+ return commands;
+ }
+
+ public void addLocalResource(String subPath, LocalResource resource) {
+ localResources.put(subPath, resource);
+ }
+
+ public void addLocalResource(String subPath, LocalResource resource, String mountPath) {
+ localResources.put(subPath, resource);
+ mountPaths.put(subPath, mountPath);
+ }
+
+ /**
+ * Accessor to the credentials
+ * @return the credentials associated with this launcher
+ */
+ public Credentials getCredentials() {
+ return credentials;
+ }
+
+
+ public void addCommand(String cmd) {
+ commands.add(cmd);
+ }
+
+ /**
+ * Complete the launch context (copy in env vars, etc).
+ * @return the container to launch
+ */
+ public ContainerLaunchContext completeContainerLaunch() throws IOException {
+
+ String cmdStr = SliderUtils.join(commands, " ", false);
+ log.debug("Completed setting up container command {}", cmdStr);
+ containerLaunchContext.setCommands(commands);
+
+ //env variables
+ if (log.isDebugEnabled()) {
+ log.debug("Environment variables");
+ for (Map.Entry<String, String> envPair : envVars.entrySet()) {
+ log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
+ }
+ }
+ containerLaunchContext.setEnvironment(envVars);
+
+ //service data
+ if (log.isDebugEnabled()) {
+ log.debug("Service Data size");
+ for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) {
+ log.debug("\"{}\"=> {} bytes of data", entry.getKey(),
+ entry.getValue().array().length);
+ }
+ }
+ containerLaunchContext.setServiceData(serviceData);
+
+ // resources
+ dumpLocalResources();
+ containerLaunchContext.setLocalResources(localResources);
+
+ //tokens
+ log.debug("{} tokens", credentials.numberOfTokens());
+ containerLaunchContext.setTokens(CredentialUtils.marshallCredentials(
+ credentials));
+
+ if(yarnDockerMode){
+ Map<String, String> env = containerLaunchContext.getEnvironment();
+ env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", dockerNetwork);
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME",
+ dockerHostname);
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
+ StringBuilder sb = new StringBuilder();
+ for (Entry<String,String> mount : mountPaths.entrySet()) {
+ if (sb.length() > 0) {
+ sb.append(",");
+ }
+ sb.append(mount.getKey());
+ sb.append(":");
+ sb.append(mount.getValue());
+ }
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString());
+ log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString());
+ }
+
+ return containerLaunchContext;
+ }
+
+ public void setRetryContext(int maxRetries, int retryInterval) {
+ ContainerRetryContext retryContext = ContainerRetryContext
+ .newInstance(ContainerRetryPolicy.RETRY_ON_ALL_ERRORS, null, maxRetries,
+ retryInterval);
+ containerLaunchContext.setContainerRetryContext(retryContext);
+ }
+
+ /**
+ * Dump local resources at debug level
+ */
+ private void dumpLocalResources() {
+ if (log.isDebugEnabled()) {
+ log.debug("{} resources: ", localResources.size());
+ for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
+
+ String key = entry.getKey();
+ LocalResource val = entry.getValue();
+ log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
+ }
+ }
+ }
+
+ /**
+ * This is critical for an insecure cluster -it passes
+ * down the username to YARN, and so gives the code running
+ * in containers the rights it needs to work with
+ * data.
+ * @throws IOException problems working with current user
+ */
+ protected void propagateUsernameInInsecureCluster() throws IOException {
+ //insecure cluster: propagate user name via env variable
+ String userName = UserGroupInformation.getCurrentUser().getUserName();
+ envVars.put(YarnServiceConstants.HADOOP_USER_NAME, userName);
+ }
+
+ /**
+ * Utility method to set up the classpath
+ * @param classpath classpath to use
+ */
+ public void setClasspath(ClasspathConstructor classpath) {
+ setEnv(CLASSPATH, classpath.buildClasspath());
+ }
+
+ /**
+ * Set an environment variable in the launch context
+ * @param var variable name
+ * @param value value (must be non null)
+ */
+ public void setEnv(String var, String value) {
+ Preconditions.checkArgument(var != null, "null variable name");
+ Preconditions.checkArgument(value != null, "null value");
+ envVars.put(var, value);
+ }
+
+
+ public void putEnv(Map<String, String> map) {
+ envVars.putAll(map);
+ }
+
+
+ public void setDockerImage(String dockerImage) {
+ this.dockerImage = dockerImage;
+ }
+
+ public void setDockerNetwork(String dockerNetwork) {
+ this.dockerNetwork = dockerNetwork;
+ }
+
+ public void setDockerHostname(String dockerHostname) {
+ this.dockerHostname = dockerHostname;
+ }
+
+ public void setRunPrivilegedContainer(boolean runPrivilegedContainer) {
+ if (runPrivilegedContainer) {
+ this.runPrivilegedContainer = Boolean.toString(true);
+ } else {
+ this.runPrivilegedContainer = Boolean.toString(false);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
new file mode 100644
index 0000000..22b3877
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * build a classpath -allows for entries to be injected in front of
+ * YARN classpath as well as behind, adds appropriate separators,
+ * extraction of local classpath, etc.
+ */
+public class ClasspathConstructor {
+
+ public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
+ private final List<String> pathElements = new ArrayList<>();
+
+ public ClasspathConstructor() {
+ }
+
+
+ /**
+ * Get the list of JARs from the YARN settings
+ * @param config configuration
+ */
+ public List<String> yarnApplicationClasspath(Configuration config) {
+ String[] cp = config.getTrimmedStrings(
+ YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH);
+ return cp != null ? Arrays.asList(cp) : new ArrayList<String>(0);
+
+ }
+
+
+ @Override
+ public String toString() {
+ return buildClasspath();
+ }
+
+ public String buildClasspath() {
+ return SliderUtils.join(pathElements,
+ CLASS_PATH_SEPARATOR,
+ false);
+ }
+
+ /**
+ * Get a copy of the path list
+ * @return the JARs
+ */
+ public List<String> getPathElements() {
+ return Collections.unmodifiableList(pathElements);
+ }
+
+ /**
+ * Append an entry
+ * @param path path
+ */
+ public void append(String path) {
+ pathElements.add(path);
+ }
+
+ /**
+ * Insert a path at the front of the list. This places it ahead of
+ * the standard YARN artifacts
+ * @param path path to the JAR. Absolute or relative -on the target
+ * system
+ */
+ public void insert(String path) {
+ pathElements.add(0, path);
+ }
+
+ public void appendAll(Collection<String> paths) {
+ pathElements.addAll(paths);
+ }
+
+ public void insertAll(Collection<String> paths) {
+ pathElements.addAll(0, paths);
+ }
+
+
+ public void addLibDir(String pathToLibDir) {
+ append(buildLibDir(pathToLibDir));
+ }
+
+ public void insertLibDir(String pathToLibDir) {
+ insert(buildLibDir(pathToLibDir));
+ }
+
+ public void addClassDirectory(String pathToDir) {
+ append(appendDirectoryTerminator(pathToDir));
+ }
+
+ public void insertClassDirectory(String pathToDir) {
+ insert(buildLibDir(appendDirectoryTerminator(pathToDir)));
+ }
+
+
+ public void addRemoteClasspathEnvVar() {
+ append(ApplicationConstants.Environment.CLASSPATH.$$());
+ }
+
+
+ public void insertRemoteClasspathEnvVar() {
+ append(ApplicationConstants.Environment.CLASSPATH.$$());
+ }
+
+
+ /**
+ * Build a lib dir path
+ * @param pathToLibDir path to the directory; may or may not end with a
+ * trailing space
+ * @return a path to a lib dir that is compatible with the java classpath
+ */
+ public String buildLibDir(String pathToLibDir) {
+ String dir = appendDirectoryTerminator(pathToLibDir);
+ dir += "*";
+ return dir;
+ }
+
+ private String appendDirectoryTerminator(String pathToLibDir) {
+ String dir = pathToLibDir.trim();
+ if (!dir.endsWith("/")) {
+ dir += "/";
+ }
+ return dir;
+ }
+
+ /**
+ * Split a classpath. This uses the local path separator so MUST NOT
+ * be used to work with remote classpaths
+ * @param localpath local path
+ * @return a splite
+ */
+ public Collection<String> splitClasspath(String localpath) {
+ String separator = System.getProperty("path.separator");
+ return StringUtils.getStringCollection(localpath, separator);
+ }
+
+ /**
+ * Get the local JVM classpath split up
+ * @return the list of entries on the JVM classpath env var
+ */
+ public Collection<String> localJVMClasspath() {
+ return splitClasspath(System.getProperty("java.class.path"));
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
deleted file mode 100644
index 1fa07ce..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ZookeeperUtils {
- public static final int DEFAULT_PORT = 2181;
-
- public static String buildConnectionString(String zkHosts, int port) {
- String zkPort = Integer.toString(port);
- //parse the hosts
- String[] hostlist = zkHosts.split(",", 0);
- String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
- return quorum;
- }
-
- /**
- * Take a quorum list and split it to (trimmed) pairs
- * @param hostPortQuorumList list of form h1:port, h2:port2,...
- * @return a possibly empty list of values between commas. They may not be
- * valid hostname:port pairs
- */
- public static List<String> splitToPairs(String hostPortQuorumList) {
- // split an address hot
- String[] strings = StringUtils.getStrings(hostPortQuorumList);
- int len = 0;
- if (strings != null) {
- len = strings.length;
- }
- List<String> tuples = new ArrayList<String>(len);
- if (strings != null) {
- for (String s : strings) {
- tuples.add(s.trim());
- }
- }
- return tuples;
- }
-
- /**
- * Split a quorum list into a list of hostnames and ports
- * @param hostPortQuorumList split to a list of hosts and ports
- * @return a list of values
- */
- public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
- // split an address hot
- String[] strings = StringUtils.getStrings(hostPortQuorumList);
- int len = 0;
- if (strings != null) {
- len = strings.length;
- }
- List<HostAndPort> list = new ArrayList<HostAndPort>(len);
- if (strings != null) {
- for (String s : strings) {
- list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
- }
- }
- return list;
- }
-
- /**
- * Build up to a hosts only list
- * @param hostAndPorts
- * @return a list of the hosts only
- */
- public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
- StringBuilder sb = new StringBuilder();
- for (HostAndPort hostAndPort : hostAndPorts) {
- sb.append(hostAndPort.getHostText()).append(",");
- }
- if (sb.length() > 0) {
- sb.delete(sb.length() - 1, sb.length());
- }
- return sb.toString();
- }
-
- public static String buildQuorumEntry(HostAndPort hostAndPort,
- int defaultPort) {
- String s = hostAndPort.toString();
- if (hostAndPort.hasPort()) {
- return s;
- } else {
- return s + ":" + defaultPort;
- }
- }
-
- /**
- * Build a quorum list, injecting a ":defaultPort" ref if needed on
- * any entry without one
- * @param hostAndPorts
- * @param defaultPort
- * @return
- */
- public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
- List<String> entries = new ArrayList<String>(hostAndPorts.size());
- for (HostAndPort hostAndPort : hostAndPorts) {
- entries.add(buildQuorumEntry(hostAndPort, defaultPort));
- }
- return SliderUtils.join(entries, ",", false);
- }
-
- public static String convertToHostsOnlyList(String quorum) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
- return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
- }
-
- public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts =
- ZookeeperUtils.splitToHostsAndPorts(quorum);
- if (hostAndPorts.isEmpty()) {
- throw new BadConfigException("empty zookeeper quorum");
- }
- return hostAndPorts;
- }
-
- public static int getFirstPort(String quorum, int defVal) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
- int port = hostAndPorts.get(0).getPortOrDefault(defVal);
- return port;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
deleted file mode 100644
index 0a21c24..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.yarn.proto";
-option java_outer_classname = "ClientAMProtocol";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.yarn;
-
-service ClientAMProtocolService {
- rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto);
- rpc getStatus(GetStatusRequestProto) returns (GetStatusResponseProto);
- rpc stop(StopRequestProto) returns (StopResponseProto);
-}
-
-message FlexComponentsRequestProto {
- repeated ComponentCountProto components = 1;
-}
-
-message ComponentCountProto {
- optional string name = 1;
- optional int64 numberOfContainers = 2;
-}
-
-message FlexComponentsResponseProto{
-}
-
-message GetStatusRequestProto {
-
-}
-message GetStatusResponseProto {
- optional string status = 1;
-}
-
-message StopRequestProto {
-
-}
-
-message StopResponseProto {
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
deleted file mode 100644
index 4fa81ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.base.Supplier;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.component.ComponentState;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
-import static org.mockito.Mockito.mock;
-
-public class MockServiceAM extends ServiceMaster {
-
- Application application;
- // The list of containers fed by tests to be returned on
- // AMRMClientCallBackHandler#onContainersAllocated
- final List<Container> feedContainers =
- Collections.synchronizedList(new LinkedList<>());
-
- public MockServiceAM(Application application) {
- super(application.getName());
- this.application = application;
- }
-
-
- @Override
- protected ContainerId getAMContainerId()
- throws BadClusterStateException {
- return ContainerId.newContainerId(ApplicationAttemptId
- .newInstance(ApplicationId.fromString(application.getId()), 1), 1);
- }
-
- @Override
- protected Path getAppDir() {
- Path path = new Path(new Path("target", "apps"), application.getName());
- System.out.println("Application path: " + path);
- return path;
- }
-
- @Override
- protected ServiceScheduler createServiceScheduler(ServiceContext context)
- throws IOException, YarnException {
- return new ServiceScheduler(context) {
-
- @Override
- protected YarnRegistryViewForProviders createYarnRegistryOperations(
- ServiceContext context, RegistryOperations registryClient) {
- return mock(YarnRegistryViewForProviders.class);
- }
-
- @Override
- protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
- AMRMClientImpl client1 = new AMRMClientImpl() {
- @Override public AllocateResponse allocate(float progressIndicator)
- throws YarnException, IOException {
-
- AllocateResponse.AllocateResponseBuilder builder =
- AllocateResponse.newBuilder();
- synchronized (feedContainers) {
- if (feedContainers.isEmpty()) {
- System.out.println("Allocating........ no containers");
- return builder.build();
- } else {
- // The AMRMClient will return containers for compoenent that are
- // at FLEXING state
- List<Container> allocatedContainers = new LinkedList<>();
- Iterator<Container> itor = feedContainers.iterator();
- while (itor.hasNext()) {
- Container c = itor.next();
- org.apache.hadoop.yarn.service.component.Component component =
- componentsById.get(c.getAllocationRequestId());
- if (component.getState() == ComponentState.FLEXING) {
- System.out.println("Allocated container " + c.getId());
- allocatedContainers.add(c);
- itor.remove();
- }
- }
- return builder.allocatedContainers(allocatedContainers).build();
- }
- }
- }
-
- @Override
- public RegisterApplicationMasterResponse registerApplicationMaster(
- String appHostName, int appHostPort, String appTrackingUrl) {
- return mock(RegisterApplicationMasterResponse.class);
- }
-
- @Override public void unregisterApplicationMaster(
- FinalApplicationStatus appStatus, String appMessage,
- String appTrackingUrl) {
- // DO nothing
- }
- };
-
- return AMRMClientAsync
- .createAMRMClientAsync(client1, 1000,
- this.new AMRMClientCallback());
- }
-
- @Override
- public NMClientAsync createNMClient() {
- NMClientAsync nmClientAsync = super.createNMClient();
- nmClientAsync.setClient(mock(NMClient.class));
- return nmClientAsync;
- }
- };
- }
-
- @Override protected void loadApplicationJson(ServiceContext context,
- SliderFileSystem fs) throws IOException {
- context.application = application;
- }
-
- /**
- *
- * @param application The application for the component
- * @param id The id for the container
- * @param compName The component to which the container is fed
- * @return
- */
- public Container feedContainerToComp(Application application, int id,
- String compName) {
- ApplicationId applicationId = ApplicationId.fromString(application.getId());
- ContainerId containerId = ContainerId
- .newContainerId(ApplicationAttemptId.newInstance(applicationId, 1), id);
- NodeId nodeId = NodeId.newInstance("localhost", 1234);
- Container container = Container
- .newInstance(containerId, nodeId, "localhost",
- Resource.newInstance(100, 1), Priority.newInstance(0), null);
-
- long allocateId =
- context.scheduler.getAllComponents().get(compName).getAllocateId();
- container.setAllocationRequestId(allocateId);
- synchronized (feedContainers) {
- feedContainers.add(container);
- }
- return container;
- }
-
- public void flexComponent(String compName, long numberOfContainers)
- throws IOException {
- ClientAMProtocol.ComponentCountProto componentCountProto =
- ClientAMProtocol.ComponentCountProto.newBuilder().setName(compName)
- .setNumberOfContainers(numberOfContainers).build();
- ClientAMProtocol.FlexComponentsRequestProto requestProto =
- ClientAMProtocol.FlexComponentsRequestProto.newBuilder()
- .addComponents(componentCountProto).build();
- context.clientAMService.flexComponents(requestProto);
- }
-
- public Component getComponent(String compName) {
- return context.scheduler.getAllComponents().get(compName);
- }
-
- public void waitForDependenciesSatisfied(String compName)
- throws TimeoutException, InterruptedException {
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- return context.scheduler.getAllComponents().get(compName)
- .areDependenciesReady();
- }
- }, 1000, 20000);
- }
-
- public void waitForNumDesiredContainers(String compName,
- int numDesiredContainers) throws TimeoutException, InterruptedException {
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- return context.scheduler.getAllComponents().get(compName)
- .getNumDesiredInstances() == numDesiredContainers;
- }
- }, 1000, 20000);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
deleted file mode 100644
index 73172bf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-
-public class ServiceTestUtils {
-
- public static final JsonSerDeser<Application> JSON_SER_DESER =
- new JsonSerDeser<>(Application.class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
-
- // Example service definition
- // 2 components, each of which has 2 containers.
- protected Application createExampleApplication() {
- Application exampleApp = new Application();
- exampleApp.setName("example-app");
- exampleApp.addComponent(createComponent("compa"));
- exampleApp.addComponent(createComponent("compb"));
- return exampleApp;
- }
-
- protected Component createComponent(String name) {
- return createComponent(name, 2L, "sleep 1000");
- }
-
- protected Component createComponent(String name, long numContainers,
- String command) {
- Component comp1 = new Component();
- comp1.setNumberOfContainers(numContainers);
- comp1.setLaunchCommand(command);
- comp1.setName(name);
- Resource resource = new Resource();
- comp1.setResource(resource);
- resource.setMemory("128");
- resource.setCpus(1);
- return comp1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
deleted file mode 100644
index 1a22875..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
-import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Test for ServiceApiUtil helper methods.
- */
-public class TestServiceApiUtil {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestServiceApiUtil.class);
- private static final String EXCEPTION_PREFIX = "Should have thrown " +
- "exception: ";
- private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
- "exception: ";
-
- private static final String LEN_64_STR =
- "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
-
- private static final YarnConfiguration CONF_DEFAULT_DNS = new
- YarnConfiguration();
- private static final YarnConfiguration CONF_DNS_ENABLED = new
- YarnConfiguration();
-
- @BeforeClass
- public static void init() {
- CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
- }
-
- @Test(timeout = 90000)
- public void testResourceValidation() throws Exception {
- assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
- .length());
-
- SliderFileSystem sfs = initMock(null);
-
- Application app = new Application();
-
- // no name
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no name");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
- }
-
- // bad format name
- String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
- for (String badName : badNames) {
- app.setName(badName);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- ERROR_APPLICATION_NAME_INVALID_FORMAT, badName), e.getMessage());
- }
- }
-
- // launch command not specified
- app.setName(LEN_64_STR);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
- Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
- } catch (IllegalArgumentException e) {
- assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
- e.getMessage());
- }
-
- // launch command not specified
- app.setName(LEN_64_STR.substring(0, RegistryConstants
- .MAX_FQDN_LABEL_LENGTH));
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
- } catch (IllegalArgumentException e) {
- assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
- e.getMessage());
- }
-
- // resource not specified
- app.setLaunchCommand("sleep 3600");
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no resource");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
- // memory not specified
- Resource res = new Resource();
- app.setResource(res);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no memory");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
- // invalid no of cpus
- res.setMemory("100mb");
- res.setCpus(-2);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(
- EXCEPTION_PREFIX + "application with invalid no of cpus");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
- // number of containers not specified
- res.setCpus(2);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no container count");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .contains(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // specifying profile along with cpus/memory raises exception
- res.setProfile("hbase_finance_large");
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX
- + "application with resource profile along with cpus/memory");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(RestApiErrorMessages
- .ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
- DEFAULT_COMPONENT_NAME),
- e.getMessage());
- }
-
- // currently resource profile alone is not supported.
- // TODO: remove the next test once resource profile alone is supported.
- res.setCpus(null);
- res.setMemory(null);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with resource profile only");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
- e.getMessage());
- }
-
- // unset profile here and add cpus/memory back
- res.setProfile(null);
- res.setCpus(2);
- res.setMemory("2gb");
-
- // null number of containers
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "null number of containers");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // negative number of containers
- app.setNumberOfContainers(-1L);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // everything valid here
- app.setNumberOfContainers(5L);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- LOG.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
- }
-
- @Test
- public void testArtifacts() throws IOException {
- SliderFileSystem sfs = initMock(null);
-
- Application app = new Application();
- app.setName("name");
- Resource res = new Resource();
- app.setResource(res);
- res.setMemory("512M");
- app.setNumberOfContainers(3L);
-
- // no artifact id fails with default type
- Artifact artifact = new Artifact();
- app.setArtifact(artifact);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // no artifact id fails with APPLICATION type
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // no artifact id fails with TARBALL type
- artifact.setType(Artifact.TypeEnum.TARBALL);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // everything valid here
- artifact.setType(Artifact.TypeEnum.DOCKER);
- artifact.setId("docker.io/centos:centos7");
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- LOG.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- // defaults assigned
- assertEquals(app.getComponents().get(0).getName(),
- DEFAULT_COMPONENT_NAME);
- assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
- }
-
- private static Resource createValidResource() {
- Resource res = new Resource();
- res.setMemory("512M");
- return res;
- }
-
- private static Component createValidComponent(String compName) {
- Component comp = new Component();
- comp.setName(compName);
- comp.setResource(createValidResource());
- comp.setNumberOfContainers(1L);
- return comp;
- }
-
- private static Application createValidApplication(String compName) {
- Application app = new Application();
- app.setLaunchCommand("sleep 3600");
- app.setName("name");
- app.setResource(createValidResource());
- app.setNumberOfContainers(1L);
- if (compName != null) {
- app.addComponent(createValidComponent(compName));
- }
- return app;
- }
-
- private static SliderFileSystem initMock(Application ext) throws IOException {
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
- .class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- if (ext != null) {
- expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
- .anyTimes();
- }
- replay(sfs, mockFs, jsonSerDeser);
- ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
- return sfs;
- }
-
- @Test
- public void testExternalApplication() throws IOException {
- Application ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
-
- Application app = createValidApplication(null);
-
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("id");
- app.setArtifact(artifact);
-
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- assertEquals(1, app.getComponents().size());
- assertNotNull(app.getComponent("comp1"));
- }
-
- @Test
- public void testDuplicateComponents() throws IOException {
- SliderFileSystem sfs = initMock(null);
-
- String compName = "comp1";
- Application app = createValidApplication(compName);
- app.addComponent(createValidComponent(compName));
-
- // duplicate component name fails
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with component collision");
- } catch (IllegalArgumentException e) {
- assertEquals("Component name collision: " + compName, e.getMessage());
- }
- }
-
- @Test
- public void testExternalDuplicateComponent() throws IOException {
- Application ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
-
- Application app = createValidApplication("comp1");
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("id");
- app.getComponent("comp1").setArtifact(artifact);
-
- // duplicate component name okay in the case of APPLICATION component
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
- }
-
- @Test
- public void testExternalComponent() throws IOException {
- Application ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
-
- Application app = createValidApplication("comp2");
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("id");
- app.setArtifact(artifact);
-
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- assertEquals(1, app.getComponents().size());
- // artifact ID not inherited from global
- assertNotNull(app.getComponent("comp2"));
-
- // set APPLICATION artifact id on component
- app.getComponent("comp2").setArtifact(artifact);
-
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- assertEquals(1, app.getComponents().size());
- // original component replaced by external component
- assertNotNull(app.getComponent("comp1"));
- }
-
- public static void verifyDependencySorting(List<Component> components,
- Component... expectedSorting) {
- Collection<Component> actualSorting = ServiceApiUtil.sortByDependencies(
- components);
- assertEquals(expectedSorting.length, actualSorting.size());
- int i = 0;
- for (Component component : actualSorting) {
- assertEquals(expectedSorting[i++], component);
- }
- }
-
- @Test
- public void testDependencySorting() throws IOException {
- Component a = new Component().name("a");
- Component b = new Component().name("b");
- Component c = new Component().name("c");
- Component d = new Component().name("d").dependencies(Arrays.asList("c"));
- Component e = new Component().name("e").dependencies(Arrays.asList("b",
- "d"));
-
- verifyDependencySorting(Arrays.asList(a, b, c), a, b, c);
- verifyDependencySorting(Arrays.asList(c, a, b), c, a, b);
- verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e);
- verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e);
-
- c.setDependencies(Arrays.asList("e"));
- try {
- verifyDependencySorting(Arrays.asList(a, b, c, d, e));
- Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle");
- } catch (IllegalArgumentException ex) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d,
- e)), ex.getMessage());
- }
-
- SliderFileSystem sfs = initMock(null);
- Application application = createValidApplication(null);
- application.setComponents(Arrays.asList(c, d, e));
- try {
- ServiceApiUtil.validateAndResolveApplication(application, sfs,
- CONF_DEFAULT_DNS);
- Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies");
- } catch (IllegalArgumentException ex) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex
- .getMessage());
- }
- }
-
- @Test
- public void testInvalidComponent() throws IOException {
- SliderFileSystem sfs = initMock(null);
- testComponent(sfs);
- }
-
- @Test
- public void testValidateCompName() {
- String[] invalidNames = {
- "EXAMPLE", // UPPER case not allowed
- "example_app" // underscore not allowed.
- };
- for (String name : invalidNames) {
- try {
- ServiceApiUtil.validateCompName(name);
- Assert.fail();
- } catch (IllegalArgumentException ex) {
- ex.printStackTrace();
- }
- }
- }
-
- private static void testComponent(SliderFileSystem sfs)
- throws IOException {
- int maxLen = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
- assertEquals(19, Long.toString(Long.MAX_VALUE).length());
- maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
-
- String compName = LEN_64_STR.substring(0, maxLen + 1);
- Application app = createValidApplication(null);
- app.addComponent(createValidComponent(compName));
-
- // invalid component name fails if dns is enabled
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with invalid component name");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(RestApiErrorMessages
- .ERROR_COMPONENT_NAME_INVALID, maxLen, compName), e.getMessage());
- }
-
- // does not fail if dns is disabled
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- compName = LEN_64_STR.substring(0, maxLen);
- app = createValidApplication(null);
- app.addComponent(createValidComponent(compName));
-
- // does not fail
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
deleted file mode 100644
index a36e0b4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.curator.test.TestingCluster;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
-import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
-import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-
-/**
- * End to end tests to test deploying services with MiniYarnCluster and a in-JVM
- * ZK testing cluster.
- */
-public class TestYarnNativeServices extends ServiceTestUtils{
-
- private static final Log LOG =
- LogFactory.getLog(TestYarnNativeServices.class);
-
- private MiniYARNCluster yarnCluster = null;
- private MiniDFSCluster hdfsCluster = null;
- private FileSystem fs = null;
- protected Configuration conf = null;
- private static final int NUM_NMS = 1;
- private File basedir;
-
- @Rule
- public TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @Before
- public void setup() throws Exception {
- setupInternal(NUM_NMS);
- }
-
- private void setupInternal(int numNodeManager)
- throws Exception {
- LOG.info("Starting up YARN cluster");
-// Logger rootLogger = LogManager.getRootLogger();
-// rootLogger.setLevel(Level.DEBUG);
- conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
- // reduce the teardown waiting time
- conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
- conf.set("yarn.log.dir", "target");
- // mark if we need to launch the v1 timeline server
- // disable aux-service based timeline aggregators
- conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
- conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
- // Enable ContainersMonitorImpl
- conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
- LinuxResourceCalculatorPlugin.class.getName());
- conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
- ProcfsBasedProcessTree.class.getName());
- conf.setBoolean(
- YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true);
- conf.setBoolean(TIMELINE_SERVICE_ENABLED, false);
- conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
- conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000);
- conf.setLong(AM_RESOURCE_MEM, 526);
- conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5);
- // Disable vmem check to disallow NM killing the container
- conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
- conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
- // setup zk cluster
- TestingCluster zkCluster;
- zkCluster = new TestingCluster(1);
- zkCluster.start();
- conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
- conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
- LOG.info("ZK cluster: " + zkCluster.getConnectString());
-
- fs = FileSystem.get(conf);
- basedir = new File("target", "apps");
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
-
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
-
- if (yarnCluster == null) {
- yarnCluster =
- new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1,
- numNodeManager, 1, 1);
- yarnCluster.init(conf);
- yarnCluster.start();
-
- waitForNMsToRegister();
-
- URL url = Thread.currentThread().getContextClassLoader()
- .getResource("yarn-site.xml");
- if (url == null) {
- throw new RuntimeException(
- "Could not find 'yarn-site.xml' dummy file in classpath");
- }
- Configuration yarnClusterConfig = yarnCluster.getConfig();
- yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- new File(url.getPath()).getParent());
- //write the document to a buffer (not directly to the file, as that
- //can cause the file being written to get read -which will then fail.
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- yarnClusterConfig.writeXml(bytesOut);
- bytesOut.close();
- //write the bytes to the file in the classpath
- OutputStream os = new FileOutputStream(new File(url.getPath()));
- os.write(bytesOut.toByteArray());
- os.close();
- LOG.info("Write yarn-site.xml configs to: " + url);
- }
- if (hdfsCluster == null) {
- HdfsConfiguration hdfsConfig = new HdfsConfiguration();
- hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
- .numDataNodes(1).build();
- }
-
- try {
- Thread.sleep(2000);
- } catch (InterruptedException e) {
- LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
- }
-
-
- }
-
- private void waitForNMsToRegister() throws Exception {
- int sec = 60;
- while (sec >= 0) {
- if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size()
- >= NUM_NMS) {
- break;
- }
- Thread.sleep(1000);
- sec--;
- }
- }
-
- @After
- public void tearDown() throws IOException {
- if (yarnCluster != null) {
- try {
- yarnCluster.stop();
- } finally {
- yarnCluster = null;
- }
- }
- if (hdfsCluster != null) {
- try {
- hdfsCluster.shutdown();
- } finally {
- hdfsCluster = null;
- }
- }
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- SliderFileSystem sfs = new SliderFileSystem(conf);
- Path appDir = sfs.getBaseApplicationPath();
- sfs.getFileSystem().delete(appDir, true);
- }
-
-
-
- // End-to-end test to use ServiceClient to deploy a service.
- // 1. Create a service with 2 components, each of which has 2 containers
- // 2. Flex up each component to 3 containers and check the component instance names
- // 3. Flex down each component to 1 container and check the component instance names
- // 4. Flex up each component to 2 containers and check the component instance names
- // 5. Stop the service
- // 6. Destroy the service
- @Test (timeout = 200000)
- public void testCreateFlexStopDestroyService() throws Exception {
- ServiceClient client = createClient();
- Application exampleApp = createExampleApplication();
- client.actionCreate(exampleApp);
- SliderFileSystem fileSystem = new SliderFileSystem(conf);
- Path appDir = fileSystem.buildClusterDirPath(exampleApp.getName());
- // check app.json is persisted.
- Assert.assertTrue(
- fs.exists(new Path(appDir, exampleApp.getName() + ".json")));
- waitForAllCompToBeReady(client, exampleApp);
-
- // Flex two components, each from 2 container to 3 containers.
- flexComponents(client, exampleApp, 3L);
- // wait for flex to be completed, increase from 2 to 3 containers.
- waitForAllCompToBeReady(client, exampleApp);
- // check all instances name for each component are in sequential order.
- checkCompInstancesInOrder(client, exampleApp);
-
- // flex down to 1
- flexComponents(client, exampleApp, 1L);
- waitForAllCompToBeReady(client, exampleApp);
- checkCompInstancesInOrder(client, exampleApp);
-
- // check component dir and registry are cleaned up.
-
- // flex up again to 2
- flexComponents(client, exampleApp, 2L);
- waitForAllCompToBeReady(client, exampleApp);
- checkCompInstancesInOrder(client, exampleApp);
-
- // stop the service
- LOG.info("Stop the service");
- client.actionStop(exampleApp.getName(), true);
- ApplicationReport report = client.getYarnClient()
- .getApplicationReport(ApplicationId.fromString(exampleApp.getId()));
- // AM unregisters with RM successfully
- Assert.assertEquals(FINISHED, report.getYarnApplicationState());
- Assert.assertEquals(FinalApplicationStatus.ENDED,
- report.getFinalApplicationStatus());
-
- LOG.info("Destroy the service");
- //destroy the service and check the app dir is deleted from fs.
- client.actionDestroy(exampleApp.getName());
- // check the application dir on hdfs (in this case, local fs) are deleted.
- Assert.assertFalse(fs.exists(appDir));
- }
-
- // Create compa with 2 containers
- // Create compb with 2 containers which depends on compa
- // Check containers for compa started before containers for compb
- @Test (timeout = 200000)
- public void testComponentStartOrder() throws Exception {
- ServiceClient client = createClient();
- Application exampleApp = new Application();
- exampleApp.setName("teststartorder");
- exampleApp.addComponent(createComponent("compa", 2, "sleep 1000"));
- Component compb = createComponent("compb", 2, "sleep 1000");
-
- // Let compb depedends on compa;
- compb.setDependencies(Collections.singletonList("compa"));
- exampleApp.addComponent(compb);
-
- client.actionCreate(exampleApp);
- waitForAllCompToBeReady(client, exampleApp);
-
- // check that containers for compa are launched before containers for compb
- checkContainerLaunchDependencies(client, exampleApp, "compa", "compb");
-
- client.actionStop(exampleApp.getName(), true);
- client.actionDestroy(exampleApp.getName());
- }
-
- // Check containers launched are in dependency order
- // Get all containers into a list and sort based on container launch time e.g.
- // compa-c1, compa-c2, compb-c1, compb-c2;
- // check that the container's launch time are align with the dependencies.
- private void checkContainerLaunchDependencies(ServiceClient client,
- Application exampleApp, String... compOrder)
- throws IOException, YarnException {
- Application retrievedApp = client.getStatus(exampleApp.getName());
- List<Container> containerList = new ArrayList<>();
- for (Component component : retrievedApp.getComponents()) {
- containerList.addAll(component.getContainers());
- }
- // sort based on launchTime
- containerList
- .sort((o1, o2) -> o1.getLaunchTime().compareTo(o2.getLaunchTime()));
- LOG.info("containerList: " + containerList);
- // check the containers are in the dependency order.
- int index = 0;
- for (String comp : compOrder) {
- long num = retrievedApp.getComponent(comp).getNumberOfContainers();
- for (int i = 0; i < num; i++) {
- String compInstanceName = containerList.get(index).getComponentName();
- String compName =
- compInstanceName.substring(0, compInstanceName.lastIndexOf('-'));
- Assert.assertEquals(comp, compName);
- index++;
- }
- }
- }
-
-
- private Map<String, Long> flexComponents(ServiceClient client,
- Application exampleApp, long count) throws YarnException, IOException {
- Map<String, Long> compCounts = new HashMap<>();
- compCounts.put("compa", count);
- compCounts.put("compb", count);
- // flex will update the persisted conf to reflect latest number of containers.
- exampleApp.getComponent("compa").setNumberOfContainers(count);
- exampleApp.getComponent("compb").setNumberOfContainers(count);
- client.flexByRestService(exampleApp.getName(), compCounts);
- return compCounts;
- }
-
- // Check each component's comp instances name are in sequential order.
- // E.g. If there are two instances compA-1 and compA-2
- // When flex up to 4 instances, it should be compA-1 , compA-2, compA-3, compA-4
- // When flex down to 3 instances, it should be compA-1 , compA-2, compA-3.
- private void checkCompInstancesInOrder(ServiceClient client,
- Application exampleApp) throws IOException, YarnException {
- Application application = client.getStatus(exampleApp.getName());
- for (Component comp : application.getComponents()) {
- checkEachCompInstancesInOrder(comp);
- }
- }
-
- private void checkRegistryAndCompDirDeleted() {
-
- }
-
- private void checkEachCompInstancesInOrder(Component component) {
- long expectedNumInstances = component.getNumberOfContainers();
- Assert.assertEquals(expectedNumInstances, component.getContainers().size());
- TreeSet<String> instances = new TreeSet<>();
- for (Container container : component.getContainers()) {
- instances.add(container.getComponentName());
- }
-
- int i = 0;
- for (String s : instances) {
- Assert.assertEquals(component.getName() + "-" + i, s);
- i++;
- }
- }
-
- private void waitForOneCompToBeReady(ServiceClient client,
- Application exampleApp, String readyComp)
- throws TimeoutException, InterruptedException {
- long numExpectedContainers =
- exampleApp.getComponent(readyComp).getNumberOfContainers();
- GenericTestUtils.waitFor(() -> {
- try {
- Application retrievedApp = client.getStatus(exampleApp.getName());
- Component retrievedComp = retrievedApp.getComponent(readyComp);
-
- if (retrievedComp.getContainers() != null
- && retrievedComp.getContainers().size() == numExpectedContainers) {
- LOG.info(readyComp + " found " + numExpectedContainers
- + " containers running");
- return true;
- } else {
- LOG.info(" Waiting for " + readyComp + "'s containers to be running");
- return false;
- }
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }, 2000, 200000);
- }
-
- // wait until all the containers for all components become ready state
- private void waitForAllCompToBeReady(ServiceClient client,
- Application exampleApp) throws TimeoutException, InterruptedException {
- int expectedTotalContainers = countTotalContainers(exampleApp);
- GenericTestUtils.waitFor(() -> {
- try {
- Application retrievedApp = client.getStatus(exampleApp.getName());
- int totalReadyContainers = 0;
- LOG.info("Num Components " + retrievedApp.getComponents().size());
- for (Component component : retrievedApp.getComponents()) {
- LOG.info("looking for " + component.getName());
- LOG.info(component);
- if (component.getContainers() != null) {
- if (component.getContainers().size() == exampleApp
- .getComponent(component.getName()).getNumberOfContainers()) {
- for (Container container : component.getContainers()) {
- LOG.info(
- "Container state " + container.getState() + ", component "
- + component.getName());
- if (container.getState() == ContainerState.READY) {
- totalReadyContainers++;
- LOG.info("Found 1 ready container " + container.getId());
- }
- }
- } else {
- LOG.info(component.getName() + " Expected number of containers "
- + exampleApp.getComponent(component.getName())
- .getNumberOfContainers() + ", current = " + component
- .getContainers());
- }
- }
- }
- LOG.info("Exit loop, totalReadyContainers= " + totalReadyContainers
- + " expected = " + expectedTotalContainers);
- return totalReadyContainers == expectedTotalContainers;
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }, 2000, 200000);
- }
-
- private ServiceClient createClient() throws Exception {
- ServiceClient client = new ServiceClient() {
- @Override protected Path addJarResource(String appName,
- Map<String, LocalResource> localResources)
- throws IOException, SliderException {
- // do nothing, the Unit test will use local jars
- return null;
- }
- };
- client.init(conf);
- client.start();
- return client;
- }
-
-
- private int countTotalContainers(Application application) {
- int totalContainers = 0;
- for (Component component : application.getComponents()) {
- totalContainers += component.getNumberOfContainers();
- }
- return totalContainers;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
deleted file mode 100644
index a22c000..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-
-/**
- * Test for building / resolving components of type APPLICATION.
- */
-public class TestBuildExternalComponents {
-
- protected Configuration conf = new YarnConfiguration();
- private File basedir;
-
- // Check component names match with expected
- private static void checkComponentNames(List<Component> components,
- Set<String> expectedComponents) {
- Assert.assertEquals(expectedComponents.size(), components.size());
- for (Component comp : components) {
- Assert.assertTrue(expectedComponents.contains(comp.getName()));
- }
- }
-
- // 1. Build the appDef and store on fs
- // 2. check component names
- private void buildAndCheckComponents(String appName, String appDef,
- SliderFileSystem sfs, Set<String> names) throws Throwable {
- String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- ServiceCLI cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient();
- client.init(conf);
- client.start();
- }
- };
- cli.exec(clientArgs);
-
- // verify generated conf
- List<Component> components =
- ServiceApiUtil.getApplicationComponents(sfs, appName);
- checkComponentNames(components, names);
- }
-
- @Before
- public void setup() throws IOException {
- basedir = new File("target", "apps");
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
- }
-
- @After
- public void tearDown() throws IOException {
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- }
-
- // Test applications defining external components(APPLICATION type)
- // can be resolved correctly
- @Test
- public void testExternalComponentBuild() throws Throwable {
- SliderFileSystem sfs = new SliderFileSystem(conf);
-
- Set<String> nameSet = new HashSet<>();
- nameSet.add("simple");
- nameSet.add("master");
- nameSet.add("worker");
-
- // app-1 has 3 components: simple, master, worker
- buildAndCheckComponents("app-1", ExampleAppJson.APP_JSON, sfs, nameSet);
- buildAndCheckComponents("external-0", ExampleAppJson.EXTERNAL_JSON_0, sfs,
- nameSet);
-
- nameSet.add("other");
-
- // external1 has 3 components: simple(APPLICATION - app1), master and other
- buildAndCheckComponents("external-1", ExampleAppJson.EXTERNAL_JSON_1, sfs,
- nameSet);
-
- nameSet.add("another");
-
- // external2 has 2 components: ext(APPLICATION - external1), another
- buildAndCheckComponents("external-2", ExampleAppJson.EXTERNAL_JSON_2, sfs,
- nameSet);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
deleted file mode 100644
index 20c06ab..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Mockito.mock;
-
-public class TestServiceCLI {
-
- protected Configuration conf = new YarnConfiguration();
- private File basedir;
- private ServiceCLI cli;
- private SliderFileSystem fs;
-
- private void buildApp(String appName, String appDef) throws Throwable {
- String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- cli.exec(clientArgs);
- }
-
- @Before
- public void setup() throws Throwable {
- basedir = new File("target", "apps");
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
- fs = new SliderFileSystem(conf);
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
-
- // create a CLI and skip connection to AM
- cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient() {
- @Override protected ClientAMProtocol getAMProxy(String appName,
- ApplicationReport report) throws IOException {
- return mock(ClientAMProtocol.class);
- }
- @Override protected ClientAMProtocol getAMProxy(String appName)
- throws IOException, YarnException {
- return mock(ClientAMProtocol.class);
- }
- };
- client.init(conf);
- client.start();
- }
- };
- }
-
- @After
- public void tearDown() throws IOException {
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- }
-
- // Test flex components count are persisted.
- @Test
- public void testFlexComponents() throws Throwable {
- buildApp("service-1", ExampleAppJson.APP_JSON);
-
- checkCompCount("master", 1L);
-
- // increase by 2
- String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"};
- ClientArgs clientArgs = new ClientArgs(flexUpArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
-
- checkCompCount("master", 3L);
-
- // decrease by 1
- String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"};
- clientArgs = new ClientArgs(flexDownArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
-
- checkCompCount("master", 2L);
-
- String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"};
- clientArgs = new ClientArgs(flexAbsoluteArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
-
- checkCompCount("master", 10L);
- }
-
- private void checkCompCount(String compName, long count) throws IOException {
- List<Component> components =
- ServiceApiUtil.getApplicationComponents(fs, "service-1");
- for (Component component : components) {
- if (component.getName().equals(compName)) {
- Assert.assertEquals(count, component.getNumberOfContainers().longValue());
- return;
- }
- }
- Assert.fail();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
deleted file mode 100644
index 9e13200..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-
-/**
- * Names of the example configs.
- */
-public final class ExampleAppJson {
-
- public static final String APP_JSON = "app.json";
- public static final String OVERRIDE_JSON = "app-override.json";
- public static final String DEFAULT_JSON = "default.json";
- public static final String EXTERNAL_JSON_0 = "external0.json";
- public static final String EXTERNAL_JSON_1 = "external1.json";
- public static final String EXTERNAL_JSON_2 = "external2.json";
-
- public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/";
-
-
- private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON,
- DEFAULT_JSON};
-
- public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>();
- static {
- for (String example : ALL_EXAMPLES) {
- ALL_EXAMPLE_RESOURCES.add(PACKAGE + example);
- }
- }
-
- private ExampleAppJson() {
- }
-
- public static Application loadResource(String name) throws IOException {
- return JSON_SER_DESER.fromResource(PACKAGE + name);
- }
-
- public static String resourceName(String name) {
- return "target/test-classes" + PACKAGE + name;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
deleted file mode 100644
index 954d117..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test global configuration resolution.
- */
-public class TestAppJsonResolve extends Assert {
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestAppJsonResolve.class);
-
- @Test
- public void testOverride() throws Throwable {
- Application orig = ExampleAppJson.loadResource(OVERRIDE_JSON);
-
- Configuration global = orig.getConfiguration();
- assertEquals("a", global.getProperty("g1"));
- assertEquals("b", global.getProperty("g2"));
- assertEquals(2, global.getFiles().size());
-
- Configuration simple = orig.getComponent("simple").getConfiguration();
- assertEquals(0, simple.getProperties().size());
- assertEquals(1, simple.getFiles().size());
-
- Configuration master = orig.getComponent("master").getConfiguration();
- assertEquals("m", master.getProperty("name"));
- assertEquals("overridden", master.getProperty("g1"));
- assertEquals(0, master.getFiles().size());
-
- Configuration worker = orig.getComponent("worker").getConfiguration();
- LOG.info("worker = {}", worker);
- assertEquals(3, worker.getProperties().size());
- assertEquals(0, worker.getFiles().size());
-
- assertEquals("worker", worker.getProperty("name"));
- assertEquals("overridden-by-worker", worker.getProperty("g1"));
- assertNull(worker.getProperty("g2"));
- assertEquals("1000", worker.getProperty("timeout"));
-
- // here is the resolution
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
- ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
- YarnConfiguration());
-
- global = orig.getConfiguration();
- LOG.info("global = {}", global);
- assertEquals("a", global.getProperty("g1"));
- assertEquals("b", global.getProperty("g2"));
- assertEquals(2, global.getFiles().size());
-
- simple = orig.getComponent("simple").getConfiguration();
- assertEquals(2, simple.getProperties().size());
- assertEquals("a", simple.getProperty("g1"));
- assertEquals("b", simple.getProperty("g2"));
- assertEquals(2, simple.getFiles().size());
-
- Set<ConfigFile> files = new HashSet<>();
- Map<String, String> props = new HashMap<>();
- props.put("k1", "overridden");
- props.put("k2", "v2");
- files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
- files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
- assertTrue(files.contains(simple.getFiles().get(0)));
- assertTrue(files.contains(simple.getFiles().get(1)));
-
- master = orig.getComponent("master").getConfiguration();
- LOG.info("master = {}", master);
- assertEquals(3, master.getProperties().size());
- assertEquals("m", master.getProperty("name"));
- assertEquals("overridden", master.getProperty("g1"));
- assertEquals("b", master.getProperty("g2"));
- assertEquals(2, master.getFiles().size());
-
- props.put("k1", "v1");
- files.clear();
- files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
- files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
-
- assertTrue(files.contains(master.getFiles().get(0)));
- assertTrue(files.contains(master.getFiles().get(1)));
-
- worker = orig.getComponent("worker").getConfiguration();
- LOG.info("worker = {}", worker);
- assertEquals(4, worker.getProperties().size());
-
- assertEquals("worker", worker.getProperty("name"));
- assertEquals("overridden-by-worker", worker.getProperty("g1"));
- assertEquals("b", worker.getProperty("g2"));
- assertEquals("1000", worker.getProperty("timeout"));
- assertEquals(2, worker.getFiles().size());
-
- assertTrue(files.contains(worker.getFiles().get(0)));
- assertTrue(files.contains(worker.getFiles().get(1)));
- }
-
- @Test
- public void testOverrideExternalConfiguration() throws IOException {
- Application orig = ExampleAppJson.loadResource(EXTERNAL_JSON_1);
-
- Configuration global = orig.getConfiguration();
- assertEquals(0, global.getProperties().size());
-
- assertEquals(3, orig.getComponents().size());
-
- Configuration simple = orig.getComponent("simple").getConfiguration();
- assertEquals(0, simple.getProperties().size());
-
- Configuration master = orig.getComponent("master").getConfiguration();
- assertEquals(1, master.getProperties().size());
- assertEquals("is-overridden", master.getProperty("g3"));
-
- Configuration other = orig.getComponent("other").getConfiguration();
- assertEquals(0, other.getProperties().size());
-
- // load the external application
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
- Application ext = ExampleAppJson.loadResource(APP_JSON);
- ServiceApiUtil.validateAndResolveApplication(ext, sfs, new
- YarnConfiguration());
- reset(sfs, mockFs);
-
- // perform the resolution on original application
- JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
- .class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
- .anyTimes();
- replay(sfs, mockFs, jsonSerDeser);
- ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
- ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
- YarnConfiguration());
-
- global = orig.getConfiguration();
- assertEquals(0, global.getProperties().size());
-
- assertEquals(4, orig.getComponents().size());
-
- simple = orig.getComponent("simple").getConfiguration();
- assertEquals(3, simple.getProperties().size());
- assertEquals("a", simple.getProperty("g1"));
- assertEquals("b", simple.getProperty("g2"));
- assertEquals("60",
- simple.getProperty("yarn.service.failure-count-reset.window"));
-
- master = orig.getComponent("master").getConfiguration();
- assertEquals(5, master.getProperties().size());
- assertEquals("512M", master.getProperty("jvm.heapsize"));
- assertEquals("overridden", master.getProperty("g1"));
- assertEquals("b", master.getProperty("g2"));
- assertEquals("is-overridden", master.getProperty("g3"));
- assertEquals("60",
- simple.getProperty("yarn.service.failure-count-reset.window"));
-
- Configuration worker = orig.getComponent("worker").getConfiguration();
- LOG.info("worker = {}", worker);
- assertEquals(4, worker.getProperties().size());
- assertEquals("512M", worker.getProperty("jvm.heapsize"));
- assertEquals("overridden-by-worker", worker.getProperty("g1"));
- assertEquals("b", worker.getProperty("g2"));
- assertEquals("60",
- worker.getProperty("yarn.service.failure-count-reset.window"));
-
- other = orig.getComponent("other").getConfiguration();
- assertEquals(0, other.getProperties().size());
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/58] [abbrv] hadoop git commit: YARN-7201. Added an apache httpd
example YARN service. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-7201. Added an apache httpd example YARN service. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c70e5df1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c70e5df1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c70e5df1
Branch: refs/heads/yarn-native-services
Commit: c70e5df100ba800f7170f44f63d388ae056cb74d
Parents: ce74e64
Author: Jian He <ji...@apache.org>
Authored: Mon Sep 25 16:36:43 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:16 2017 -0800
----------------------------------------------------------------------
.../examples/httpd-no-dns/httpd-no-dns.json | 62 ++++++++
.../httpd-no-dns/httpd-proxy-no-dns.conf | 24 +++
.../examples/httpd/httpd-proxy.conf | 24 +++
.../examples/httpd/httpd.json | 55 +++++++
.../yarn/service/api/ServiceApiConstants.java | 5 +
.../yarn/service/component/Component.java | 31 ++++
.../yarn/service/provider/ProviderUtils.java | 1 +
.../hadoop/registry/server/dns/RegistryDNS.java | 4 +
.../src/site/markdown/yarn-service/Examples.md | 159 +++++++++++++++++++
.../src/site/markdown/yarn-service/Overview.md | 3 +-
10 files changed, 367 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
new file mode 100644
index 0000000..6b35538
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
@@ -0,0 +1,62 @@
+{
+ "name": "httpd-service-no-dns",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "httpd",
+ "number_of_containers": 2,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "readiness_check": {
+ "type": "HTTP",
+ "props": {
+ "url": "http://${THIS_HOST}:8080"
+ }
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "ENV",
+ "dest_file": "/var/www/html/index.html",
+ "props": {
+ "content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "httpd-proxy",
+ "number_of_containers": 1,
+ "dependencies": [ "httpd" ],
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "TEMPLATE",
+ "dest_file": "/etc/httpd/conf.d/httpd-proxy.conf",
+ "src_file": "httpd-proxy-no-dns.conf"
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "Apache HTTP Server": "http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080"
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
new file mode 100644
index 0000000..9894e64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+<Proxy balancer://test>
+ BalancerMember http://${HTTPD-0_IP}:8080
+ BalancerMember http://${HTTPD-1_IP}:8080
+ ProxySet lbmethod=bytraffic
+</Proxy>
+
+ProxyPass "/" "balancer://test/"
+ProxyPassReverse "/" "balancer://test/"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd-proxy.conf
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd-proxy.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd-proxy.conf
new file mode 100644
index 0000000..e8651a5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd-proxy.conf
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+<Proxy balancer://test>
+ BalancerMember http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080
+ BalancerMember http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080
+ ProxySet lbmethod=bytraffic
+</Proxy>
+
+ProxyPass "/" "balancer://test/"
+ProxyPassReverse "/" "balancer://test/"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
new file mode 100644
index 0000000..e63376d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
@@ -0,0 +1,55 @@
+{
+ "name": "httpd-service",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "httpd",
+ "number_of_containers": 2,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "ENV",
+ "dest_file": "/var/www/html/index.html",
+ "props": {
+ "content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "httpd-proxy",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "TEMPLATE",
+ "dest_file": "/etc/httpd/conf.d/httpd-proxy.conf",
+ "src_file": "httpd-proxy.conf"
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "Apache HTTP Server": "http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080"
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
index 0bfb220..a85191c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
@@ -52,6 +52,11 @@ public interface ServiceApiConstants {
String CONTAINER_ID = $("CONTAINER_ID");
+ // Templates for component instance host/IP
+ String COMPONENT_HOST = $("%s_HOST");
+
+ String COMPONENT_IP = $("%s_IP");
+
// Constants for default cluster ZK
String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index cb7131e..cbaf472 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -47,8 +47,10 @@ import org.slf4j.LoggerFactory;
import java.text.MessageFormat;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -58,6 +60,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
+import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.START;
import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP;
@@ -386,6 +389,34 @@ public class Component implements EventHandler<ComponentEvent> {
return true;
}
+ public Map<String, String> getDependencyHostIpTokens() {
+ Map<String, String> tokens = new HashMap<>();
+ List<String> dependencies = componentSpec.getDependencies();
+ if (SliderUtils.isEmpty(dependencies)) {
+ return tokens;
+ }
+ for (String dependency : dependencies) {
+ Collection<ComponentInstance> instances = scheduler.getAllComponents()
+ .get(dependency).getAllComponentInstances().values();
+ for (ComponentInstance instance : instances) {
+ if (instance.getContainerStatus() == null) {
+ continue;
+ }
+ if (SliderUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
+ SliderUtils.isUnset(instance.getContainerStatus().getHost())) {
+ continue;
+ }
+ String ip = instance.getContainerStatus().getIPs().get(0);
+ String host = instance.getContainerStatus().getHost();
+ tokens.put(String.format(COMPONENT_IP,
+ instance.getCompInstanceName().toUpperCase()), ip);
+ tokens.put(String.format(COMPONENT_HOST,
+ instance.getCompInstanceName().toUpperCase()), host);
+ }
+ }
+ return tokens;
+ }
+
private void incRunningContainers() {
componentMetrics.containersRunning.incr();
scheduler.getServiceMetrics().containersRunning.incr();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index ec0c2ca..93abd73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -397,6 +397,7 @@ public class ProviderUtils implements YarnServiceConstants {
tokens.put(CONTAINER_ID, instance.getContainer().getId().toString());
tokens.put(COMPONENT_ID,
String.valueOf(instance.getCompInstanceId().getId()));
+ tokens.putAll(instance.getComponent().getDependencyHostIpTokens());
return tokens;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 9ffc9db..51139be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -1550,6 +1550,10 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
private final RegistryCommand removeRecordCommand = new RegistryCommand() {
@Override
public void exec(Zone zone, Record record) throws IOException {
+ if (zone == null) {
+ LOG.error("Unable to remove record because zone is null: {}", record);
+ return;
+ }
zone.removeRecord(record);
LOG.info("Removed {}", record);
if (isDNSSECEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
new file mode 100644
index 0000000..db6d553
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -0,0 +1,159 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN Service Examples
+
+This document describes some example service definitions (`Yarnfile`).
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+## Apache web server - httpd (with registry DNS)
+Below is the `Yarnfile` for a service called `httpd-service` with two `httpd` instances.
+There is also an httpd proxy instance (httpd-proxy-0) that proxies between the other two httpd instances (httpd-0 and httpd-1).
+
+Note this example requires registry DNS.
+```
+{
+ "name": "httpd-service",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "httpd",
+ "number_of_containers": 2,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "ENV",
+ "dest_file": "/var/www/html/index.html",
+ "props": {
+ "content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "httpd-proxy",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "centos/httpd-24-centos7:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/bin/run-httpd",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "files": [
+ {
+ "type": "TEMPLATE",
+ "dest_file": "/etc/httpd/conf.d/httpd-proxy.conf",
+ "src_file": "httpd-proxy.conf"
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "Apache HTTP Server": "http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080"
+ }
+}
+```
+This `Yarnfile` is already included in the Hadoop distribution, along with the required configuration template `httpd-proxy.conf`.
+First upload the configuration template file to HDFS:
+```
+hdfs dfs -copyFromLocal ${HADOOP_YARN_HOME}/share/hadoop/yarn/yarn-service-examples/httpd/httpd-proxy.conf .
+```
+
+The proxy configuration template looks like the following and will configure the httpd-proxy-0 container to balance between the httpd-0 and httpd-1 containers evenly:
+```
+<Proxy balancer://test>
+ BalancerMember http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080
+ BalancerMember http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080
+ ProxySet lbmethod=bytraffic
+</Proxy>
+
+ProxyPass "/" "balancer://test/"
+ProxyPassReverse "/" "balancer://test/"
+```
+
+Then run the service with the command:
+```
+yarn service create [service-name] --example httpd
+```
+where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
+
+Once the service is running, navigate to `http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` to see the root page.
+The pages should alternately show "Hello from httpd-0!" or "Hello from httpd-1!"
+
+The individual httpd URLs can also be visited, `http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` and `http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080`.
+
+If unsure of your hostnames, visit the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
+
+## Apache web server - httpd (without registry DNS)
+
+A similar IP-based example is provided for environments that do not have registry DNS set up.
+The service name for this example is `httpd-service-no-dns`.
+There are a couple of additions to the `Yarnfile` for the `httpd-service` described above.
+A readiness check is added for the `httpd` component:
+```
+ "readiness_check": {
+ "type": "HTTP",
+ "props": {
+ "url": "http://${THIS_HOST}:8080"
+ }
+ },
+```
+and `httpd` is added as a dependency for the `httpd-proxy` component:
+```
+ "dependencies": [ "httpd" ],
+```
+
+This means that the httpd-proxy-0 instance will not be started until after an HTTP probe has succeeded for the httpd-0 and httpd-1 containers.
+This is necessary so that the IPs of the containers can be used in the configuration of httpd-proxy-0.
+The proxy configuration is similar to that of the previous example, with the BalancerMember lines changed as follows:
+```
+ BalancerMember http://${HTTPD-0_IP}:8080
+ BalancerMember http://${HTTPD-1_IP}:8080
+```
+
+Note that IP and HOST variables such as `${HTTPD-0_IP}` and `${HTTPD-0_HOST}` should only be used by a component that has a dependency on the named component (`httpd` in this case) AND should only be used when the named component specifies a readiness check.
+Here, `httpd-proxy` has a dependency on `httpd` and `httpd` has an HTTP readiness check.
+Without the dependency and readiness check, the httpd-proxy-0 container would be started in parallel with the httpd-0 and http-1 containers, and the IPs and hosts would not be assigned yet for httpd-0 and httpd-1.
+
+Other variables can be used by any component.
+
+Before creating the service, upload the proxy configuration to HDFS:
+```
+hdfs dfs -copyFromLocal ${HADOOP_YARN_HOME}/share/hadoop/yarn/yarn-service-examples/httpd-no-dns/httpd-proxy-no-dns.conf .
+```
+
+Then run the service with the command:
+```
+yarn service create [service-name] --example httpd-no-dns
+```
+where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
+
+Look up your IPs at the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
+Then visit port 8080 for each IP to view the pages.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
index 3dd4d8c..407fbc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -52,7 +52,8 @@ The benefits of combining these workloads are two-fold:
* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN.
* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN.
-* [Service Discovery](ServiceDiscovery.md): Deep dives into the YARN DNS internals
+* [Service Discovery](ServiceDiscovery.md): Deep dives into the YARN DNS internals.
+* [Examples](Examples.md): List some example service definitions (`Yarnfile`).
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
deleted file mode 100644
index c2866cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-import java.io.File;
-
-/**
- * Registry actions
- *
- * --instance {app name}, if a / is in it, refers underneath?
- * --dest {destfile}
- * --list : list instances of slider service
- * --listfiles
- */
-@Parameters(commandNames = {ACTION_REGISTRY},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-
-public class ActionRegistryArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_REGISTRY
- + " ("
- + Arguments.ARG_LIST + "|"
- + Arguments.ARG_LISTCONF + "|"
- + Arguments.ARG_LISTEXP + "|"
- + Arguments.ARG_LISTFILES + "|"
- + Arguments.ARG_GETCONF + "|"
- + Arguments.ARG_GETEXP + "> "
- + Arguments.ARG_NAME + " <name> "
- + " )"
- + "[" + Arguments.ARG_VERBOSE + "] "
- + "[" + Arguments.ARG_USER + "] "
- + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
- + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
- + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
- + System.getProperty("line.separator")
- + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
- ;
- public ActionRegistryArgs() {
- }
-
- public ActionRegistryArgs(String name) {
- this.name = name;
- }
-
- @Override
- public String getActionName() {
- return ACTION_REGISTRY;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_LISTCONF},
- description = "list configurations")
- public boolean listConf;
-
- @Parameter(names = {ARG_GETCONF},
- description = "get configuration")
- public String getConf;
-
- @Parameter(names = {ARG_LISTEXP},
- description = "list exports")
- public boolean listExports;
-
- @Parameter(names = {ARG_GETEXP},
- description = "get export")
- public String getExport;
-
- @Parameter(names = {ARG_LISTFILES},
- description = "list files")
- public String listFiles;
-
- @Parameter(names = {ARG_GETFILES},
- description = "get files")
- public String getFiles;
-
- //--format
- @Parameter(names = ARG_FORMAT,
- description = "Format for a response: <xml|json|properties>")
- public String format = ConfigFormat.XML.toString() ;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
- description = "Output destination")
- public File out;
-
- @Parameter(names = {ARG_NAME},
- description = "name of an instance")
- public String name;
-
- @Parameter(names = {ARG_SERVICETYPE},
- description = "optional service type")
- public String serviceType = YarnServiceConstants.APP_TYPE;
-
- @Parameter(names = {ARG_VERBOSE},
- description = "verbose output")
- public boolean verbose;
-
- @Parameter(names = {ARG_INTERNAL},
- description = "fetch internal registry entries")
- public boolean internal;
-
- @Parameter(names = {ARG_USER},
- description = "the name of the user whose application is being resolved")
- public String user;
-
- /**
- * validate health of all the different operations
- * @throws BadCommandArgumentsException
- */
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
-
- //verify that at most one of the operations is set
- int gets = s(getConf) + s(getFiles) + s(getExport);
- int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
- int set = lists + gets;
- if (set > 1) {
- throw new UsageException(USAGE);
- }
-
- if (out != null && ( set == 0)) {
- throw new UsageException("output path"
- + " is only supported on 'get' operations: ");
- }
- if (!list && !is(name)) {
- throw new UsageException("Argument " + ARG_NAME
- +" missing: ");
-
- }
- }
-
- private int s(String arg) {
- return is(arg) ? 1 : 0;
- }
-
- private boolean is(String arg) {
- return arg != null;
- }
-
- private int s(boolean arg) {
- return arg ? 1 : 0;
- }
-
- private String ifdef(String arg, boolean val) {
- return val ? (arg + " "): "";
- }
-
- private String ifdef(String arg, String val) {
- if (is(val)) {
- return arg + " " + val + " ";
- } else {
- return "";
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_REGISTRY);
- sb.append(' ');
- sb.append(ifdef(ARG_LIST, list));
- sb.append(ifdef(ARG_LISTCONF, listConf));
- sb.append(ifdef(ARG_LISTFILES, listFiles));
- sb.append(ifdef(ARG_GETCONF, getConf));
- sb.append(ifdef(ARG_GETFILES, getFiles));
-
- sb.append(ifdef(ARG_NAME, name));
- sb.append(ifdef(ARG_SERVICETYPE, serviceType));
-
-
- sb.append(ifdef(ARG_VERBOSE, verbose));
- sb.append(ifdef(ARG_INTERNAL, internal));
-
- if (out != null) {
- sb.append(ifdef(ARG_OUTPUT, out.toString()));
- }
- sb.append(ifdef(ARG_FORMAT, format));
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
deleted file mode 100644
index 65f0472..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-
-/**
- * Resolve registry entries
- *
- * --path {path}
- * --out {destfile}
- * --verbose
- * --list
- */
-@Parameters(commandNames = {ACTION_RESOLVE},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-public class ActionResolveArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_RESOLVE
- + " "
- + ARG_PATH + " <path> "
- + "[" + ARG_LIST + "] "
- + "[" + ARG_OUTPUT + " <filename> ] "
- + "[" + ARG_DESTDIR + " <directory> ] "
- ;
- public ActionResolveArgs() {
- }
-
- @Override
- public String getActionName() {
- return ACTION_RESOLVE;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_PATH},
- description = "resolve a path")
- public String path;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "destination directory for operations")
- public File destdir;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "dest file")
- public File out;
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_RESOLVE).append(" ");
- sb.append(ARG_PATH).append(" ").append(path).append(" ");
- if (list) {
- sb.append(ARG_LIST).append(" ");
- }
- if (destdir != null) {
- sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
- }
- if (out != null) {
- sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
- }
- return sb.toString();
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (StringUtils.isEmpty(path)) {
- throw new BadCommandArgumentsException("Missing mandatory argument "
- + ARG_PATH);
- }
- if (list && out != null) {
- throw new BadCommandArgumentsException("Argument "
- + ARG_OUTPUT +
- " not supported for " + ARG_LIST);
- }
- if (out != null && destdir != null) {
- throw new BadCommandArgumentsException(
- ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
- );
- }
- }
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public boolean isList() {
- return list;
- }
-
- public void setList(boolean list) {
- this.list = list;
- }
-
- public File getDestdir() {
- return destdir;
- }
-
- public void setDestdir(File destdir) {
- this.destdir = destdir;
- }
-
- public File getOut() {
- return out;
- }
-
- public void setOut(File out) {
- this.out = out;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
deleted file mode 100644
index b03dc92..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_RESOURCE},
- commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
-
-public class ActionResourceArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_RESOURCE;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install the resource(s)")
- public boolean install;
-
- @Parameter(names = {ARG_DELETE},
- description = "Delete the file")
- public boolean delete;
-
- @Parameter(names = {ARG_LIST},
- description = "List of installed files")
- public boolean list;
-
- @Parameter(names = {ARG_RESOURCE},
- description = "Name of the file or directory")
- public String resource;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "The name of the folder in which to store the resources")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
deleted file mode 100644
index 622e77d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_STATUS},
- commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
-
-public class ActionStatusArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STATUS;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for the status information")
- public String output;
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Lifetime of the application from the time of request")
- public boolean lifetime;
-
- public String getOutput() {
- return output;
- }
-
- public void setOutput(String output) {
- this.output = output;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
deleted file mode 100644
index 2b90479..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_START },
- commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
-public class ActionThawArgs extends AbstractActionArgs implements
- WaitTimeAccessor,
- LaunchArgsAccessor {
-
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_START;
- }
-
- @Override
- public int getWaittime() {
- return launchArgs.getWaittime();
- }
-
- @ParametersDelegate
- LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Life time of the application since application started at"
- + " running state")
- public long lifetime;
-
- @Override
- public String getRmAddress() {
- return launchArgs.getRmAddress();
- }
-
- @Override
- public void setWaittime(int waittime) {
- launchArgs.setWaittime(waittime);
- }
-
-
- @Override
- public File getOutputFile() {
- return launchArgs.getOutputFile();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
deleted file mode 100644
index cf48513..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_TOKENS},
- commandDescription = "save tokens to a file or list tokens in a file")
-public class ActionTokensArgs extends AbstractActionArgs {
-
- public static final String DUPLICATE_ARGS = "Only one of " +
- ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
-
- public static final String MISSING_KT_PROVIDER =
- "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
- + " must be provided";
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_TOKENS;
- }
-
- @Parameter(names = {ARG_OUTPUT},
- description = "File to write")
- public File output;
-
- @Parameter(names = {ARG_SOURCE},
- description = "source file")
- public File source;
-
- @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
- public File keytab;
-
- @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
- public String principal="";
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (output != null && source != null) {
- throw new BadCommandArgumentsException(DUPLICATE_ARGS);
- }
-
- // this is actually a !xor
- if (keytab != null ^ !principal.isEmpty()) {
- throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
deleted file mode 100644
index e310f45..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
-
-public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_UPDATE;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
deleted file mode 100644
index 00151f4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Static argument manipulation operations
- */
-public class ArgOps {
-
- private static final Logger
- log = LoggerFactory.getLogger(ArgOps.class);
-
- /**
- * create a 3-tuple
- */
- public static List<Object> triple(String msg, int min, int max) {
- List<Object> l = new ArrayList<>(3);
- l.add(msg);
- l.add(min);
- l.add(max);
- return l;
- }
-
- public static void applyFileSystemBinding(String filesystemBinding,
- Configuration conf) {
- if (filesystemBinding != null) {
- //filesystem argument was set -this overwrites any defaults in the
- //configuration
- FileSystem.setDefaultUri(conf, filesystemBinding);
- }
- }
-
- public static void splitPairs(Collection<String> pairs,
- Map<String, String> dest) {
- for (String prop : pairs) {
- String[] keyval = prop.split("=", 2);
- if (keyval.length == 2) {
- dest.put(keyval[0], keyval[1]);
- }
- }
- }
-
-
- public static void applyDefinitions(Map<String, String> definitionMap,
- Configuration conf) {
- for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- log.debug("configuration[{}]<=\"{}\"", key, val);
- conf.set(key, val, "command line");
- }
- }
-
- /**
- * Create a map from a tuple list like ['worker','2','master','1] into a map
- * ['worker':'2',"master":'1'];
- * Duplicate entries also trigger errors
- * @param description description for errors
- * @param list list to conver to tuples
- * @return the map of key value pairs -unordered.
- * @throws BadCommandArgumentsException odd #of arguments received
- */
- public static Map<String, String> convertTupleListToMap(String description,
- List<String> list) throws
- BadCommandArgumentsException {
- Map<String, String> results = new HashMap<>();
- if (list != null && !list.isEmpty()) {
- int size = list.size();
- if (size % 2 != 0) {
- //odd number of elements, not permitted
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_PARSE_FAILURE + description);
- }
- for (int count = 0; count < size; count += 2) {
- String key = list.get(count);
- String val = list.get(count + 1);
- if (results.get(key) != null) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_DUPLICATE_ENTRY + description
- + ": " + key);
- }
- results.put(key, val);
- }
- }
- return results;
- }
-
- /**
- * Create a map from a tuple list like
- * ['worker','heapsize','5G','master','heapsize','2M'] into a map
- * ['worker':'2',"master":'1'];
- * Duplicate entries also trigger errors
-
- * @throws BadCommandArgumentsException odd #of arguments received
- */
- public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
- List<String> list) throws BadCommandArgumentsException {
-
- Map<String, Map<String, String>> results = new HashMap<>();
- if (list != null && !list.isEmpty()) {
- int size = list.size();
- if (size % 3 != 0) {
- //wrong number of elements, not permitted
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_PARSE_FAILURE + description);
- }
- for (int count = 0; count < size; count += 3) {
- String role = list.get(count);
- String key = list.get(count + 1);
- String val = list.get(count + 2);
- Map<String, String> roleMap = results.get(role);
- if (roleMap == null) {
- //demand create new role map
- roleMap = new HashMap<>();
- results.put(role, roleMap);
- }
- if (roleMap.get(key) != null) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_DUPLICATE_ENTRY + description
- + ": for key " + key + " under " + role);
- }
- roleMap.put(key, val);
- }
- }
- return results;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
deleted file mode 100644
index 204149b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Here are all the arguments that may be parsed by the client or server
- * command lines.
- *
- * Important: Please keep the main list in alphabetical order
- * so it is easier to see what arguments are there
- */
-public interface Arguments {
-
- String ARG_APPDEF = "--appdef";
- String ARG_BASE_PATH = "--basepath";
- String ARG_COMPONENT = "--component";
- String ARG_COMPONENT_SHORT = "--comp";
- String ARG_COMPONENTS = "--components";
- String ARG_COMP_OPT= "--compopt";
- String ARG_COMP_OPT_SHORT = "--co";
- String ARG_CONFIG = "--config";
- String ARG_CONTAINERS = "--containers";
- String ARG_DEBUG = "--debug";
- String ARG_DEFINE = "-D";
- String ARG_DELETE = "--delete";
- String ARG_DEST = "--dest";
- String ARG_DESTDIR = "--destdir";
- String ARG_FILESYSTEM = "--fs";
- String ARG_FILESYSTEM_LONG = "--filesystem";
- String ARG_FOLDER = "--folder";
- String ARG_FORCE = "--force";
- String ARG_FORMAT = "--format";
- String ARG_GETCONF = "--getconf";
- String ARG_GETEXP = "--getexp";
- String ARG_GETFILES = "--getfiles";
- String ARG_HELP = "--help";
- String ARG_IMAGE = "--image";
- String ARG_INSTALL = "--install";
- String ARG_INTERNAL = "--internal";
- String ARG_KEYLEN = "--keylen";
- String ARG_KEYTAB = "--keytab";
- String ARG_KEYTABINSTALL = ARG_INSTALL;
- String ARG_KEYTABDELETE = ARG_DELETE;
- String ARG_KEYTABLIST = "--list";
- String ARG_LIST = "--list";
- String ARG_LISTCONF = "--listconf";
- String ARG_LISTEXP = "--listexp";
- String ARG_LISTFILES = "--listfiles";
- String ARG_LIVE = "--live";
- String ARG_MANAGER = "--manager";
- String ARG_MANAGER_SHORT = "--m";
- String ARG_MESSAGE = "--message";
- String ARG_NAME = "--name";
- String ARG_OPTION = "--option";
- String ARG_OPTION_SHORT = "-O";
- String ARG_OUTPUT = "--out";
- String ARG_OUTPUT_SHORT = "-o";
- String ARG_OVERWRITE = "--overwrite";
- String ARG_PACKAGE = "--package";
- String ARG_PATH = "--path";
- String ARG_PRINCIPAL = "--principal";
- String ARG_QUEUE = "--queue";
- String ARG_LIFETIME = "--lifetime";
- String ARG_RESOURCE = "--resource";
- String ARG_RESOURCE_MANAGER = "--rm";
- String ARG_SECURE = "--secure";
- String ARG_SERVICETYPE = "--servicetype";
- String ARG_SERVICES = "--services";
- String ARG_SOURCE = "--source";
- String ARG_STATE = "--state";
- String ARG_SYSPROP = "-S";
- String ARG_USER = "--user";
- String ARG_UPLOAD = "--upload";
- String ARG_VERBOSE = "--verbose";
- String ARG_VERSION = "--version";
- String ARG_WAIT = "--wait";
-/*
- STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
- RIGHT PLACE IN THE LIST
- */
-
- /**
- * server: URI for the cluster
- */
- String ARG_CLUSTER_URI = "-cluster-uri";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
deleted file mode 100644
index 7b957fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-
-import java.util.Collection;
-
-/**
- * Client CLI Args
- */
-
-public class ClientArgs extends CommonArgs {
-
- // =========================================================
- // Keep all of these in alphabetical order. Thanks.
- // =========================================================
-
- private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
- private final ActionClientArgs actionClientArgs = new ActionClientArgs();
- private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
- private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
- private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
- private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
- private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
- private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
- private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
- private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
- private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
- private final ActionListArgs actionListArgs = new ActionListArgs();
- private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
- private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
- private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
- private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
- private final ActionThawArgs actionThawArgs = new ActionThawArgs();
- private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
- private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
-
- public ClientArgs(String[] args) {
- super(args);
- }
-
- public ClientArgs(Collection args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
-
- addActions(
- actionBuildArgs,
- actionCreateArgs,
- actionDependencyArgs,
- actionDestroyArgs,
- actionFlexArgs,
- actionFreezeArgs,
- actionHelpArgs,
- actionStatusArgs,
- actionThawArgs
- );
- }
-
- @Override
- public void applyDefinitions(Configuration conf) throws
- BadCommandArgumentsException {
- super.applyDefinitions(conf);
- //RM
- if (getManager() != null) {
- log.debug("Setting RM to {}", getManager());
- conf.set(YarnConfiguration.RM_ADDRESS, getManager());
- }
- if (getBasePath() != null) {
- log.debug("Setting basePath to {}", getBasePath());
- conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH,
- getBasePath().toString());
- }
- }
-
-
- public ActionBuildArgs getActionBuildArgs() {
- return actionBuildArgs;
- }
-
- public ActionUpdateArgs getActionUpdateArgs() {
- return actionUpdateArgs;
- }
-
- public ActionCreateArgs getActionCreateArgs() {
- return actionCreateArgs;
- }
-
- public ActionDependencyArgs getActionDependencyArgs() {
- return actionDependencyArgs;
- }
-
- public ActionDestroyArgs getActionDestroyArgs() {
- return actionDestroyArgs;
- }
-
- public ActionExistsArgs getActionExistsArgs() {
- return actionExistsArgs;
- }
-
- public ActionFlexArgs getActionFlexArgs() {
- return actionFlexArgs;
- }
-
- public ActionFreezeArgs getActionFreezeArgs() {
- return actionFreezeArgs;
- }
-
- public ActionListArgs getActionListArgs() {
- return actionListArgs;
- }
-
-
- public ActionRegistryArgs getActionRegistryArgs() {
- return actionRegistryArgs;
- }
-
- public ActionResolveArgs getActionResolveArgs() {
- return actionResolveArgs;
- }
-
- public ActionResourceArgs getActionResourceArgs() {
- return actionResourceArgs;
- }
-
- public ActionStatusArgs getActionStatusArgs() {
- return actionStatusArgs;
- }
-
- public ActionThawArgs getActionThawArgs() {
- return actionThawArgs;
- }
-
- public ActionTokensArgs getActionTokenArgs() {
- return actionTokenArgs;
- }
-
- /**
- * Look at the chosen action and bind it as the core action for the operation.
- * @throws SliderException bad argument or similar
- */
- @Override
- public void applyAction() throws SliderException {
- String action = getAction();
- if (SliderUtils.isUnset(action)) {
- action = ACTION_HELP;
- }
- switch (action) {
- case ACTION_BUILD:
- bindCoreAction(actionBuildArgs);
- break;
-
- case ACTION_CREATE:
- bindCoreAction(actionCreateArgs);
- break;
-
- case ACTION_STOP:
- bindCoreAction(actionFreezeArgs);
- break;
-
- case ACTION_START:
- bindCoreAction(actionThawArgs);
- break;
-
- case ACTION_DEPENDENCY:
- bindCoreAction(actionDependencyArgs);
- break;
-
- case ACTION_DESTROY:
- bindCoreAction(actionDestroyArgs);
- break;
-
- case ACTION_EXISTS:
- bindCoreAction(actionExistsArgs);
- break;
-
- case ACTION_FLEX:
- bindCoreAction(actionFlexArgs);
- break;
-
- case ACTION_HELP:
- bindCoreAction(actionHelpArgs);
- break;
-
- case ACTION_KDIAG:
- bindCoreAction(actionKDiagArgs);
- break;
-
- case ACTION_KEYTAB:
- bindCoreAction(actionKeytabArgs);
- break;
-
- case ACTION_LIST:
- bindCoreAction(actionListArgs);
- break;
-
- case ACTION_REGISTRY:
- bindCoreAction(actionRegistryArgs);
- break;
-
- case ACTION_RESOLVE:
- bindCoreAction(actionResolveArgs);
- break;
-
- case ACTION_RESOURCE:
- bindCoreAction(actionResourceArgs);
- break;
-
- case ACTION_STATUS:
- bindCoreAction(actionStatusArgs);
- break;
-
- case ACTION_TOKENS:
- bindCoreAction(actionTokenArgs);
- break;
-
- case ACTION_UPDATE:
- bindCoreAction(actionUpdateArgs);
- break;
-
- default:
- throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
- + " " + action);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
deleted file mode 100644
index d44412a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterDescription;
-import com.beust.jcommander.ParameterException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This class contains the common argument set for all tne entry points,
- * and the core parsing logic to verify that the action is on the list
- * of allowed actions -and that the remaining number of arguments is
- * in the range allowed
- */
-
-public abstract class CommonArgs extends ArgOps implements SliderActions,
- Arguments {
-
- protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
-
-
- private static final int DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME = 30;
-
-
- @Parameter(names = ARG_HELP, help = true)
- public boolean help;
-
-
- /**
- -D name=value
-
- Define an HBase configuration option which overrides any options in
- the configuration XML files of the image or in the image configuration
- directory. The values will be persisted.
- Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
-
- */
-
- public Map<String, String> definitionMap = new HashMap<String, String>();
- /**
- * System properties
- */
- public Map<String, String> syspropsMap = new HashMap<String, String>();
-
-
- /**
- * fields
- */
- public final JCommander commander;
- private final String[] args;
-
- private AbstractActionArgs coreAction;
-
- /**
- * get the name: relies on arg 1 being the cluster name in all operations
- * @return the name argument, null if there is none
- */
- public String getClusterName() {
- return coreAction.getClusterName();
- }
-
- protected CommonArgs(String[] args) {
- this.args = args;
- commander = new JCommander(this);
- }
-
- protected CommonArgs(Collection args) {
- List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
- this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
- commander = new JCommander(this);
- }
-
- public String usage() {
- return usage(this, null);
- }
-
- public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
- String result = null;
- StringBuilder helperMessage = new StringBuilder();
- if (commandOfInterest == null) {
- // JCommander.usage is too verbose for a command with many options like
- // slider no short version of that is found Instead, we compose our msg by
- helperMessage.append("\nUsage: service COMMAND [options]\n");
- helperMessage.append("where COMMAND is one of\n");
- for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
- helperMessage.append(String.format("\t%-"
- + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", jcommand,
- serviceArgs.commander.getCommandDescription(jcommand) + "\n"));
- }
- helperMessage
- .append("Most commands print help when invoked without parameters or with --help");
- result = helperMessage.toString();
- } else {
- helperMessage.append("\nUsage: service ").append(commandOfInterest);
- helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <application>" : "");
- helperMessage.append("\n");
- for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
- .get(commandOfInterest).getParameters()) {
- String optional = paramDesc.getParameter().required() ? " (required)"
- : " (optional)";
- String paramName = paramDesc.getParameterized().getType() == Boolean.TYPE ? paramDesc
- .getLongestName() : paramDesc.getLongestName() + " <"
- + paramDesc.getParameterized().getName() + ">";
- helperMessage.append(String.format("\t%-"
- + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", paramName,
- paramDesc.getDescription() + optional + "\n"));
- result = helperMessage.toString();
- }
- }
- return result;
- }
-
- public static String usage(CommonArgs serviceArgs) {
- return usage(serviceArgs, null);
- }
-
- /**
- * Parse routine -includes registering the action-specific argument classes
- * and postprocess it
- * @throws SliderException on any problem
- */
- public void parse() throws SliderException {
- addActionArguments();
- try {
- commander.parse(args);
- } catch (ParameterException e) {
- throw new BadCommandArgumentsException(e, "%s in %s",
- e.toString(),
- (args != null
- ? (SliderUtils.join(args,
- " ", false))
- : "[]"));
- }
- //now copy back to this class some of the attributes that are common to all
- //actions
- postProcess();
- }
-
- /**
- * Add a command
- * @param name action
- * @param arg value
- */
- protected void addAction(String name, Object arg) {
- commander.addCommand(name, arg);
- }
-
- protected void addActions(Object... actions) {
- for (Object action : actions) {
- commander.addCommand(action);
- }
- }
-
- /**
- * Override point to add a set of actions
- */
- protected void addActionArguments() {
-
- }
-
- /**
- * validate args via {@link #validate()}
- * then postprocess the arguments
- */
- public void postProcess() throws SliderException {
- applyAction();
- validate();
-
- //apply entry set
- for (Map.Entry<String, String> entry : syspropsMap.entrySet()) {
- System.setProperty(entry.getKey(), entry.getValue());
- }
- }
-
-
- /**
- * Implementors must implement their action apply routine here
- */
- public abstract void applyAction() throws SliderException;
-
-
- /**
- * Bind the core action; this extracts any attributes that are used
- * across routines
- * @param action action to bind
- */
- protected void bindCoreAction(AbstractActionArgs action) {
- coreAction = action;
-
- splitPairs(coreAction.definitions, definitionMap);
- splitPairs(coreAction.sysprops, syspropsMap);
- }
-
- /**
- * Validate the arguments against the action requested
- */
- public void validate() throws BadCommandArgumentsException, UsageException {
- if (coreAction == null) {
- throw new UsageException(ErrorStrings.ERROR_NO_ACTION + usage());
- }
- log.debug("action={}", getAction());
- // let the action validate itself
- try {
- coreAction.validate();
- } catch (BadCommandArgumentsException e) {
- String badArgMsgBuilder =
- e.getMessage() + System.lineSeparator() + usage(this,
- coreAction.getActionName());
- throw new BadCommandArgumentsException(badArgMsgBuilder);
- }
- }
-
- /**
- * Apply all the definitions on the command line to the configuration
- * @param conf config
- */
- public void applyDefinitions(Configuration conf) throws
- BadCommandArgumentsException {
- applyDefinitions(definitionMap, conf);
- }
-
-
- /**
- * If the Filesystem binding was provided, it overrides anything in
- * the configuration
- * @param conf configuration
- */
- public void applyFileSystemBinding(Configuration conf) {
- ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf);
- }
-
- public boolean isDebug() {
- return coreAction.debug;
- }
-
-
- public String getFilesystemBinding() {
- return coreAction.filesystemBinding;
- }
-
- public Path getBasePath() { return coreAction.basePath; }
-
- public String getManager() {
- return coreAction.manager;
- }
-
- public String getAction() {
- return commander.getParsedCommand();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
deleted file mode 100644
index b6cd0a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class ComponentArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * This is a listing of the roles to create
- */
- @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT},
- arity = 2,
- description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
- splitter = DontSplitArguments.class)
- public List<String> componentTuples = new ArrayList<>(0);
-
-
- /**
- * Get the role mapping (may be empty, but never null)
- * @return role mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
- return convertTupleListToMap("component", componentTuples);
- }
-
- public List<String> getComponentTuples() {
- return componentTuples;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
deleted file mode 100644
index 85de615..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.converters.IParameterSplitter;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DontSplitArguments implements IParameterSplitter {
-
- @Override
- public List<String> split(String value) {
- List<String> list = new ArrayList<>(1);
- list.add(value);
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
deleted file mode 100644
index bf194b6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import java.io.File;
-
-/**
- * Launch args for create and start and anything else that can start something
- */
-public interface LaunchArgsAccessor extends WaitTimeAccessor {
- String getRmAddress();
-
- File getOutputFile();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
deleted file mode 100644
index 157fb61..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-import java.io.File;
-
-/**
- * Any launch-time args
- */
-public class LaunchArgsDelegate extends WaitArgsDelegate implements
- LaunchArgsAccessor {
-
-
- //TODO: do we need this?
- @Parameter(names = ARG_RESOURCE_MANAGER,
- description = "Resource manager hostname:port ",
- required = false)
- private String rmAddress;
-
- @Override
- public String getRmAddress() {
- return rmAddress;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any application report")
- public File outputFile;
-
- @Override
- public File getOutputFile() {
- return outputFile;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
deleted file mode 100644
index 7972716..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Delegate for application and resource options.
- */
-public class OptionArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * Options key value.
- */
- @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
- description = ARG_OPTION + "<name> <value>",
- splitter = DontSplitArguments.class)
- public List<String> optionTuples = new ArrayList<>(0);
-
-
- /**
- * All the app component option triples.
- */
- @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
- description = "Component option " + ARG_COMP_OPT +
- " <component> <name> <option>",
- splitter = DontSplitArguments.class)
- public List<String> compOptTriples = new ArrayList<>(0);
-
- public Map<String, String> getOptionsMap() throws
- BadCommandArgumentsException {
- return convertTupleListToMap(ARG_OPTION, optionTuples);
- }
-
- /**
- * Get the role heap mapping (may be empty, but never null).
- * @return role heap mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, Map<String, String>> getCompOptionMap()
- throws BadCommandArgumentsException {
- return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
deleted file mode 100644
index 040ac64..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.converters.BaseConverter;
-import org.apache.hadoop.fs.Path;
-
-public class PathArgumentConverter extends BaseConverter<Path> {
-
- public PathArgumentConverter(String optionName) {
- super(optionName);
- }
-
- @Override
- public Path convert(String value) {
- return new Path(value);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
deleted file mode 100644
index 1c38213..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Parameters sent by the Client to the AM
- */
-public class SliderAMArgs extends CommonArgs {
-
- SliderAMCreateAction createAction = new SliderAMCreateAction();
-
- public SliderAMArgs(String[] args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
- addActions(createAction);
- }
-
- public String getImage() {
- return createAction.image;
- }
-
- /**
- * This is the URI in the FS to the Slider cluster; the conf file (and any
- * other cluster-specifics) can be picked up here
- */
- public String getAppDefPath() {
- return createAction.sliderClusterURI;
- }
-
- /**
- * Am binding is simple: there is only one action
- */
- @Override
- public void applyAction() {
- bindCoreAction(createAction);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
deleted file mode 100644
index a446665..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-import java.io.File;
-
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class SliderAMCreateAction extends AbstractActionArgs implements
- LaunchArgsAccessor {
-
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-
- @Parameter(names = ARG_IMAGE, description = "image", required = false)
- public String image;
-
- /**
- * This is the URI in the FS to the Slider cluster; the conf file (and any
- * other cluster-specifics) can be picked up here
- */
- @Parameter(names = ARG_CLUSTER_URI,
- description = "URI to the Slider cluster", required = true)
- public String sliderClusterURI;
-
- @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
-
- @Override
- public String getRmAddress() {
- return launchArgs.getRmAddress();
- }
-
- @Override
- public int getWaittime() {
- return launchArgs.getWaittime();
- }
-
- @Override
- public void setWaittime(int waittime) {
- launchArgs.setWaittime(waittime);
- }
-
- @Override
- public File getOutputFile() {
- return launchArgs.getOutputFile();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
deleted file mode 100644
index fc3c5a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Actions.
- * Only some of these are supported by specific Slider Services; they
- * are listed here to ensure the names are consistent
- */
-public interface SliderActions {
- String ACTION_BUILD = "build";
- String ACTION_CLIENT = "client";
- String ACTION_CREATE = "create";
- String ACTION_DEPENDENCY = "dependency";
- String ACTION_UPDATE = "update";
- String ACTION_UPGRADE = "upgrade";
- String ACTION_DESTROY = "destroy";
- String ACTION_EXISTS = "exists";
- String ACTION_FLEX = "flex";
- String ACTION_STOP = "stop";
- String ACTION_HELP = "help";
- String ACTION_INSTALL_KEYTAB = "install-keytab";
- String ACTION_KDIAG = "kdiag";
- String ACTION_KEYTAB = "keytab";
- String ACTION_LIST = "list";
-
- String ACTION_REGISTRY = "registry";
- String ACTION_RESOLVE = "resolve";
- String ACTION_RESOURCE = "resource";
- String ACTION_STATUS = "status";
- String ACTION_START = "start";
- String ACTION_TOKENS = "tokens";
-
- String DESCRIBE_ACTION_BUILD =
- "Build a service specification, but do not start it";
- String DESCRIBE_ACTION_CREATE =
- "Build and start a service, it's equivalent to first invoke build and then start";
- String DESCRIBE_ACTION_DEPENDENCY =
- "Yarn service framework dependency (libraries) management";
- String DESCRIBE_ACTION_UPDATE =
- "Update template for service";
- String DESCRIBE_ACTION_UPGRADE =
- "Rolling upgrade/downgrade the component/containerto a newer/previous version";
- String DESCRIBE_ACTION_DESTROY =
- "Destroy a stopped service, service must be stopped first before destroying.";
- String DESCRIBE_ACTION_EXISTS =
- "Probe for an application running";
- String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
- String DESCRIBE_ACTION_FREEZE =
- "Stop a running service";
- String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
- String DESCRIBE_ACTION_HELP = "Print help information";
- String DESCRIBE_ACTION_LIST =
- "List running services";
- String DESCRIBE_ACTION_REGISTRY =
- "Query the registry of a service";
- String DESCRIBE_ACTION_STATUS =
- "Get the status of a service";
- String DESCRIBE_ACTION_THAW =
- "Start a service with pre-built specification or a previously stopped service";
- String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore";
- String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
- String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
-
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
deleted file mode 100644
index 86f3709..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-public class WaitArgsDelegate extends AbstractArgsDelegate implements
- WaitTimeAccessor {
-
-
- //--wait [timeout]
- @Parameter(names = {ARG_WAIT},
- description = "time to wait for an action to complete")
- public int waittime = 0;
-
-
- @Override
- public int getWaittime() {
- return waittime;
- }
-
- @Override
- public void setWaittime(int waittime) {
- this.waittime = waittime;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
deleted file mode 100644
index f6afae6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-public interface WaitTimeAccessor {
- int getWaittime();
- void setWaittime(int waittime);
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/58] [abbrv] hadoop git commit: YARN-7210. Some NPE fixes in
Registry DNS. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce74e643
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce74e643
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce74e643
Branch: refs/heads/yarn-native-services
Commit: ce74e64363abc64561263be70a923ab3e67f043f
Parents: 37c9b73
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Sep 21 10:18:42 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:16 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceScheduler.java | 6 +-----
.../hadoop/yarn/service/client/ServiceClient.java | 3 +++
.../params/AbstractClusterBuildingActionArgs.java | 2 +-
.../yarn/service/client/params/Arguments.java | 1 +
.../component/instance/ComponentInstance.java | 10 +++++-----
.../service/provider/AbstractProviderService.java | 18 +++++++++++-------
.../client/binding/RegistryTypeUtils.java | 3 ++-
.../dns/ApplicationServiceRecordProcessor.java | 15 ++++++++++++++-
.../server/dns/BaseServiceRecordProcessor.java | 8 ++++----
.../registry/server/dns/TestRegistryDNS.java | 12 ++++++++----
.../runtime/DockerLinuxContainerRuntime.java | 4 ++--
.../src/site/markdown/YarnCommands.md | 1 +
12 files changed, 53 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 7b809b9..ec5f3ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -344,11 +344,7 @@ public class ServiceScheduler extends CompositeService {
attemptId.getApplicationId().toString());
serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
PersistencePolicies.APPLICATION);
- serviceRecord.description = "Yarn Service Master";
-
- serviceRecord.addExternalEndpoint(RegistryTypeUtils
- .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
- context.clientAMService.getBindAddress()));
+ serviceRecord.description = "YarnServiceMaster";
// set any provided attributes
setUserProvidedServiceRecordAttributes(service.getConfiguration(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 6890aef..a3a9fd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -170,6 +170,9 @@ public class ServiceClient extends CompositeService
if (!StringUtils.isEmpty(args.getServiceName())) {
service.setName(args.getServiceName());
}
+ if (!StringUtils.isEmpty(args.queue)) {
+ service.setQueue(args.queue);
+ }
return service;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
index 434c6d5..4ecbe9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -37,7 +37,7 @@ public abstract class AbstractClusterBuildingActionArgs
}
@Parameter(names = {
- ARG_QUEUE }, description = "Queue to submit the service")
+ ARG_QUEUE, ARG_SHORT_QUEUE}, description = "Queue to submit the service")
public String queue;
@Parameter(names = {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
index 9672621..67571e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -77,6 +77,7 @@ public interface Arguments {
String ARG_PATH = "--path";
String ARG_PRINCIPAL = "--principal";
String ARG_QUEUE = "--queue";
+ String ARG_SHORT_QUEUE = "-q";
String ARG_LIFETIME = "--lifetime";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCE_MANAGER = "--rm";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 7d6525b..3c1e48f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -59,6 +59,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import static org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes.*;
import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*;
@@ -356,12 +357,11 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
YarnRegistryViewForProviders yarnRegistry, ContainerStatus status) {
ServiceRecord record = new ServiceRecord();
String containerId = status.getContainerId().toString();
- record.set(YarnRegistryAttributes.YARN_ID, containerId);
+ record.set(YARN_ID, containerId);
record.description = getCompInstanceName();
- record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
- PersistencePolicies.CONTAINER);
- record.set("yarn:ip", status.getIPs());
- record.set("yarn:hostname", status.getHost());
+ record.set(YARN_PERSISTENCE, PersistencePolicies.CONTAINER);
+ record.set(YARN_IP, status.getIPs().get(0));
+ record.set(YARN_HOSTNAME, status.getHost());
try {
yarnRegistry
.putComponent(RegistryPathUtils.encodeYarnID(containerId), record);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 8d607ab..6ffb84d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.service.provider;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.service.api.records.Service;
@@ -91,13 +92,16 @@ public abstract class AbstractProviderService implements ProviderService,
component, tokensForSubstitution, instance, context);
// substitute launch command
- String launchCommand = ProviderUtils
- .substituteStrWithTokens(component.getLaunchCommand(),
- tokensForSubstitution);
- CommandLineBuilder operation = new CommandLineBuilder();
- operation.add(launchCommand);
- operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
- launcher.addCommand(operation.build());
+ String launchCommand = component.getLaunchCommand();
+ // docker container may have empty commands
+ if (!StringUtils.isEmpty(launchCommand)) {
+ launchCommand = ProviderUtils
+ .substituteStrWithTokens(launchCommand, tokensForSubstitution);
+ CommandLineBuilder operation = new CommandLineBuilder();
+ operation.add(launchCommand);
+ operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+ launcher.addCommand(operation.build());
+ }
// By default retry forever every 30 seconds
launcher.setRetryContext(YarnServiceConf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java
index ec59d59..05df325 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java
@@ -22,7 +22,6 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import static org.apache.hadoop.registry.client.types.AddressTypes.*;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
import org.apache.hadoop.registry.client.types.ServiceRecord;
@@ -36,6 +35,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.hadoop.registry.client.types.AddressTypes.*;
+
/**
* Static methods to work with registry types —primarily endpoints and the
* list representation of addresses.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
index e6a1b5b..0b5f724 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
@@ -18,6 +18,8 @@ package org.apache.hadoop.registry.server.dns;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.xbill.DNS.Name;
import org.xbill.DNS.Type;
@@ -32,7 +34,8 @@ import java.util.List;
*/
public class ApplicationServiceRecordProcessor extends
BaseServiceRecordProcessor {
-
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApplicationServiceRecordProcessor.class);
/**
* Create an application service record processor.
*
@@ -57,6 +60,10 @@ public class ApplicationServiceRecordProcessor extends
*/
@Override public void initTypeToInfoMapping(ServiceRecord serviceRecord)
throws Exception {
+ if (serviceRecord.external.isEmpty()) {
+ LOG.info(serviceRecord.description + ": No external endpoints defined.");
+ return;
+ }
for (int type : getRecordTypes()) {
switch (type) {
case Type.A:
@@ -309,6 +316,9 @@ public class ApplicationServiceRecordProcessor extends
throws Exception {
this.setNames(new Name[] {getServiceName()});
List<Endpoint> endpoints = serviceRecord.external;
+ if (endpoints.isEmpty()) {
+ return;
+ }
// TODO: do we need a "hostname" attribute for an application record or
// can we rely on the first endpoint record.
this.setTarget(InetAddress.getByName(
@@ -342,6 +352,9 @@ public class ApplicationServiceRecordProcessor extends
@Override protected void init(ServiceRecord serviceRecord)
throws Exception {
super.init(serviceRecord);
+ if (getTarget() == null) {
+ return;
+ }
try {
this.setTarget(getIpv6Address(getTarget()));
} catch (UnknownHostException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
index 2fe3a6c..fd5c74f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
@@ -52,8 +52,8 @@ public abstract class BaseServiceRecordProcessor
private String domain;
private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
- private static final String SLIDER_API_PREFIX =
- "classpath:org.apache.slider.";
+ private static final String YARN_SERVICE_API_PREFIX =
+ "classpath:org.apache.hadoop.yarn.service.";
private static final String HTTP_API_TYPE = "http://";
/**
@@ -425,8 +425,8 @@ public abstract class BaseServiceRecordProcessor
*/
protected String getDNSApiFragment(String api) {
String dnsApi = null;
- if (api.startsWith(SLIDER_API_PREFIX)) {
- dnsApi = api.substring(SLIDER_API_PREFIX.length());
+ if (api.startsWith(YARN_SERVICE_API_PREFIX)) {
+ dnsApi = api.substring(YARN_SERVICE_API_PREFIX.length());
} else if (api.startsWith(HTTP_API_TYPE)) {
dnsApi = "http";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
index cc839cc..ac8d939 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -69,7 +69,8 @@ public class TestRegistryDNS extends Assert {
+ " \"type\" : \"JSONServiceRecord\",\n"
+ " \"description\" : \"Slider Application Master\",\n"
+ " \"external\" : [ {\n"
- + " \"api\" : \"classpath:org.apache.slider.appmaster.ipc\",\n"
+ + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.appmaster.ipc"
+ + "\",\n"
+ " \"addressType\" : \"host/port\",\n"
+ " \"protocolType\" : \"hadoop/IPC\",\n"
+ " \"addresses\" : [ {\n"
@@ -84,7 +85,8 @@ public class TestRegistryDNS extends Assert {
+ " \"uri\" : \"http://192.168.1.5:1027\"\n"
+ " } ]\n"
+ " }, {\n"
- + " \"api\" : \"classpath:org.apache.slider.management\",\n"
+ + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.management\""
+ + ",\n"
+ " \"addressType\" : \"uri\",\n"
+ " \"protocolType\" : \"REST\",\n"
+ " \"addresses\" : [ {\n"
@@ -92,14 +94,16 @@ public class TestRegistryDNS extends Assert {
+ " } ]\n"
+ " } ],\n"
+ " \"internal\" : [ {\n"
- + " \"api\" : \"classpath:org.apache.slider.agents.secure\",\n"
+ + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.secure"
+ + "\",\n"
+ " \"addressType\" : \"uri\",\n"
+ " \"protocolType\" : \"REST\",\n"
+ " \"addresses\" : [ {\n"
+ " \"uri\" : \"https://192.168.1.5:47700/ws/v1/slider/agents\"\n"
+ " } ]\n"
+ " }, {\n"
- + " \"api\" : \"classpath:org.apache.slider.agents.oneway\",\n"
+ + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.oneway"
+ + "\",\n"
+ " \"addressType\" : \"uri\",\n"
+ " \"protocolType\" : \"REST\",\n"
+ " \"addresses\" : [ {\n"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index f945a9b..75a28e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -812,13 +812,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.executePrivilegedOperation(null, privOp, null,
null, true, false);
LOG.info("Docker inspect output for " + containerId + ": " + output);
+ // strip off quotes if any
+ output = output.replaceAll("['\"]", "");
int index = output.lastIndexOf(',');
if (index == -1) {
LOG.error("Incorrect format for ip and host");
return null;
}
- // strip off quotes if any
- output = output.replaceAll("['\"]", "");
String ips = output.substring(0, index).trim();
String host = output.substring(index+1).trim();
String[] ipAndHost = new String[2];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index caeaeb5..51e27cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -92,6 +92,7 @@ Usage `yarn service [sub-command] [service-name] [options]`
Options:
--file,-f The local path to the service definition file.
+ --queue,-q The queue to which the service is submitted.
--example,-e The name of the example service such as:
Sleeper A simple service that launches a few non-docker sleep containers on YARN.
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/58] [abbrv] hadoop git commit: YARN-7351. Fix high CPU usage
issue in RegistryDNS. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2775fbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2775fbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2775fbc
Branch: refs/heads/yarn-native-services
Commit: d2775fbccaa187729d86459542210434ce4c3b8b
Parents: 501be9b
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Oct 25 13:01:01 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:19 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/registry/server/dns/RegistryDNS.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2775fbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 37e8429..9097c46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -189,7 +189,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
LOG.info("Opening TCP and UDP channels on {} port {}", addr, port);
addNIOUDP(addr, port);
- addNIOTCP(addr, port);
+ //TODO Fix dns lookup over TCP
+// addNIOTCP(addr, port);
}
/**
@@ -770,6 +771,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
byte[] response = null;
try {
query = new Message(in);
+ LOG.info("received TCP query {}", query.getQuestion());
response = generateReply(query, ch.socket());
if (response == null) {
return;
@@ -947,7 +949,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
input.flip();
input.get(in);
query = new Message(in);
- LOG.info("{}: received query {}", remoteAddress,
+ LOG.info("{}: received UDP query {}", remoteAddress,
query.getQuestion());
response = generateReply(query, null);
if (response == null) {
@@ -960,7 +962,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
output.put(response);
output.flip();
- LOG.info("{}: sending response", remoteAddress);
+ LOG.debug("{}: sending response", remoteAddress);
channel.send(output, remoteAddress);
}
} catch (Exception e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
deleted file mode 100644
index 83c9961..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base class for factories.
- */
-public abstract class ProviderFactory {
- protected static final Logger LOG =
- LoggerFactory.getLogger(ProviderFactory.class);
-
- protected ProviderFactory() {}
-
- public abstract AbstractClientProvider createClientProvider();
-
- public abstract ProviderService createServerProvider();
-
- public static synchronized ProviderService getProviderService(Artifact
- artifact) {
- return createSliderProviderFactory(artifact).createServerProvider();
- }
-
- public static synchronized AbstractClientProvider getClientProvider(Artifact
- artifact) {
- return createSliderProviderFactory(artifact).createClientProvider();
- }
-
- /**
- * Create a provider for a specific application
- * @param artifact artifact
- * @return provider factory
- */
- public static synchronized ProviderFactory createSliderProviderFactory(
- Artifact artifact) {
- if (artifact == null || artifact.getType() == null) {
- LOG.debug("Loading service provider type default");
- return DefaultProviderFactory.getInstance();
- }
- LOG.debug("Loading service provider type {}", artifact.getType());
- switch (artifact.getType()) {
- // TODO add handling for custom types?
- // TODO handle application
- case DOCKER:
- return DockerProviderFactory.getInstance();
- case TARBALL:
- return TarballProviderFactory.getInstance();
- default:
- throw new IllegalArgumentException(String.format("Resolution error, " +
- "%s should not be passed to createSliderProviderFactory",
- artifact.getType()));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
deleted file mode 100644
index 9ef0176..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-
-import java.io.IOException;
-
-public interface ProviderService {
-
- /**
- * Set up the entire container launch context
- */
- void buildContainerLaunchContext(AbstractLauncher containerLauncher,
- Application application, ComponentInstance instance,
- SliderFileSystem sliderFileSystem, Configuration yarnConf)
- throws IOException, SliderException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
deleted file mode 100644
index a044838..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
-import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.regex.Pattern;
-
-import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
-
-/**
- * This is a factoring out of methods handy for providers. It's bonded to a log
- * at construction time.
- */
-public class ProviderUtils implements YarnServiceConstants {
-
- protected static final Logger log =
- LoggerFactory.getLogger(ProviderUtils.class);
-
-
- /**
- * Add oneself to the classpath. This does not work
- * on minicluster test runs where the JAR is not built up.
- * @param providerResources map of provider resources to add these entries to
- * @param providerClass provider to add
- * @param jarName name of the jar to use
- * @param sliderFileSystem target filesystem
- * @param tempPath path in the cluster FS for temp files
- * @param libdir relative directory to place resources
- * @param miniClusterTestRun true if minicluster is being used
- * @return true if the class was found in a JAR
- *
- * @throws FileNotFoundException if the JAR was not found and this is NOT
- * a mini cluster test run
- * @throws IOException IO problems
- * @throws SliderException any Slider problem
- */
- public static boolean addProviderJar(
- Map<String, LocalResource> providerResources,
- Class providerClass,
- String jarName,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libdir,
- boolean miniClusterTestRun) throws
- IOException,
- SliderException {
- try {
- SliderUtils.putJar(providerResources,
- sliderFileSystem,
- providerClass,
- tempPath,
- libdir,
- jarName);
- return true;
- } catch (FileNotFoundException e) {
- if (miniClusterTestRun) {
- return false;
- } else {
- throw e;
- }
- }
- }
-
- /**
- * Loads all dependency jars from the default path.
- * @param providerResources map of provider resources to add these entries to
- * @param sliderFileSystem target filesystem
- * @param tempPath path in the cluster FS for temp files
- * @param libDir relative directory to place resources
- * @param libLocalSrcDir explicitly supplied local libs dir
- * @throws IOException trouble copying to HDFS
- * @throws SliderException trouble copying to HDFS
- */
- public static void addAllDependencyJars(
- Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String libLocalSrcDir)
- throws IOException, SliderException {
- if (SliderUtils.isSet(libLocalSrcDir)) {
- File file = new File(libLocalSrcDir);
- if (!file.exists() || !file.isDirectory()) {
- throw new BadCommandArgumentsException(
- "Supplied lib src dir %s is not valid", libLocalSrcDir);
- }
- }
- SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
- libDir, libLocalSrcDir);
- }
-
- public static String substituteStrWithTokens(String content,
- Map<String, String> tokensForSubstitution) {
- for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
- content =
- content.replaceAll(Pattern.quote(token.getKey()), token.getValue());
- }
- return content;
- }
-
- // configs will be substituted by corresponding env in tokenMap
- public static void substituteMapWithTokens(Map<String, String> configs,
- Map<String, String> tokenMap) {
- for (Map.Entry<String, String> entry : configs.entrySet()) {
- String value = entry.getValue();
- if (tokenMap != null) {
- for (Map.Entry<String, String> token : tokenMap.entrySet()) {
- value =
- value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
- }
- }
- entry.setValue(value);
- }
- }
-
- /**
- * Localize the service keytabs for the application.
- * @param launcher container launcher
- * @param fileSystem file system
- * @throws IOException trouble uploading to HDFS
- */
- public void localizeServiceKeytabs(AbstractLauncher launcher,
- SliderFileSystem fileSystem, Application application) throws IOException {
-
- Configuration conf = application.getConfiguration();
- String keytabPathOnHost =
- conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
- if (SliderUtils.isUnset(keytabPathOnHost)) {
- String amKeytabName =
- conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
- String keytabDir =
- conf.getProperty(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
- // we need to localize the keytab files in the directory
- Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
- application.getName());
- boolean serviceKeytabsDeployed = false;
- if (fileSystem.getFileSystem().exists(keytabDirPath)) {
- FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
- keytabDirPath);
- LocalResource keytabRes;
- for (FileStatus keytab : keytabs) {
- if (!amKeytabName.equals(keytab.getPath().getName())
- && keytab.getPath().getName().endsWith(".keytab")) {
- serviceKeytabsDeployed = true;
- log.info("Localizing keytab {}", keytab.getPath().getName());
- keytabRes = fileSystem.createAmResource(keytab.getPath(),
- LocalResourceType.FILE);
- launcher.addLocalResource(KEYTAB_DIR + "/" +
- keytab.getPath().getName(),
- keytabRes);
- }
- }
- }
- if (!serviceKeytabsDeployed) {
- log.warn("No service keytabs for the application have been localized. "
- + "If the application requires keytabs for secure operation, "
- + "please ensure that the required keytabs have been uploaded "
- + "to the folder {}", keytabDirPath);
- }
- }
- }
-
- // 1. Create all config files for a component on hdfs for localization
- // 2. Add the config file to localResource
- public static synchronized void createConfigFileAndAddLocalResource(
- AbstractLauncher launcher, SliderFileSystem fs, Component component,
- Map<String, String> tokensForSubstitution, ComponentInstance instance,
- ServiceContext context) throws IOException {
- Path compDir =
- new Path(new Path(fs.getAppDir(), "components"), component.getName());
- Path compInstanceDir =
- new Path(compDir, instance.getCompInstanceName());
- if (!fs.getFileSystem().exists(compInstanceDir)) {
- log.info(instance.getCompInstanceId() + ": Creating dir on hdfs: " + compInstanceDir);
- fs.getFileSystem().mkdirs(compInstanceDir,
- new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
- instance.setCompInstanceDir(compInstanceDir);
- } else {
- log.info("Component instance conf dir already exists: " + compInstanceDir);
- }
-
- if (log.isDebugEnabled()) {
- log.debug("Tokens substitution for component instance: " + instance
- .getCompInstanceName() + System.lineSeparator()
- + tokensForSubstitution);
- }
-
- for (ConfigFile originalFile : component.getConfiguration().getFiles()) {
- ConfigFile configFile = originalFile.copy();
- String fileName = new Path(configFile.getDestFile()).getName();
-
- // substitute file name
- for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
- configFile.setDestFile(configFile.getDestFile()
- .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
- }
-
- Path remoteFile = new Path(compInstanceDir, fileName);
- if (!fs.getFileSystem().exists(remoteFile)) {
- log.info("Saving config file on hdfs for component " + instance
- .getCompInstanceName() + ": " + configFile);
-
- if (configFile.getSrcFile() != null) {
- // Load config file template
- switch (configFile.getType()) {
- case HADOOP_XML:
- // Hadoop_xml_template
- resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(),
- tokensForSubstitution, configFile, remoteFile, context);
- break;
- case TEMPLATE:
- // plain-template
- resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(),
- tokensForSubstitution, configFile, remoteFile, context);
- break;
- default:
- log.info("Not supporting loading src_file for " + configFile);
- break;
- }
- } else {
- // non-template
- resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
- instance, configFile, fileName, remoteFile);
- }
- }
-
- // Add resource for localization
- LocalResource configResource =
- fs.createAmResource(remoteFile, LocalResourceType.FILE);
- File destFile = new File(configFile.getDestFile());
- String symlink = APP_CONF_DIR + "/" + fileName;
- if (destFile.isAbsolute()) {
- launcher.addLocalResource(symlink, configResource,
- configFile.getDestFile());
- log.info("Add config file for localization: " + symlink + " -> "
- + configResource.getResource().getFile() + ", dest mount path: "
- + configFile.getDestFile());
- } else {
- launcher.addLocalResource(symlink, configResource);
- log.info("Add config file for localization: " + symlink + " -> "
- + configResource.getResource().getFile());
- }
- }
- }
-
- private static void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
- Map<String, String> tokensForSubstitution, ComponentInstance instance,
- ConfigFile configFile, String fileName, Path remoteFile)
- throws IOException {
- // substitute non-template configs
- substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
-
- // write configs onto hdfs
- PublishedConfiguration publishedConfiguration =
- new PublishedConfiguration(fileName,
- configFile.getProps().entrySet());
- if (!fs.getFileSystem().exists(remoteFile)) {
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(
- ConfigFormat.resolve(configFile.getType().toString()),
- publishedConfiguration);
- try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) {
- configurationOutputter.save(os);
- os.flush();
- }
- } else {
- log.info("Component instance = " + instance.getCompInstanceName()
- + ", config file already exists: " + remoteFile);
- }
- }
-
- // 1. substitute config template - only handle hadoop_xml format
- // 2. save on hdfs
- @SuppressWarnings("unchecked")
- private static void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs,
- Map<String, String> tokensForSubstitution, ConfigFile configFile,
- Path remoteFile, ServiceContext context) throws IOException {
- Map<String, String> conf;
- try {
- conf = (Map<String, String>) context.configCache.get(configFile);
- } catch (ExecutionException e) {
- log.info("Failed to load config file: " + configFile, e);
- return;
- }
- // make a copy for substitution
- org.apache.hadoop.conf.Configuration confCopy =
- new org.apache.hadoop.conf.Configuration(false);
- for (Map.Entry<String, String> entry : conf.entrySet()) {
- confCopy.set(entry.getKey(), entry.getValue());
- }
- // substitute properties
- for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
- confCopy.set(entry.getKey(), entry.getValue());
- }
- // substitute env variables
- for (Map.Entry<String, String> entry : confCopy) {
- String val = entry.getValue();
- if (val != null) {
- for (Map.Entry<String, String> token : tokensForSubstitution
- .entrySet()) {
- val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue());
- confCopy.set(entry.getKey(), val);
- }
- }
- }
- // save on hdfs
- try (OutputStream output = fs.create(remoteFile)) {
- confCopy.writeXml(output);
- log.info("Reading config from: " + configFile.getSrcFile()
- + ", writing to: " + remoteFile);
- }
- }
-
- // 1) read the template as a string
- // 2) do token substitution
- // 3) save on hdfs
- private static void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
- Map<String, String> tokensForSubstitution, ConfigFile configFile,
- Path remoteFile, ServiceContext context) {
- String content;
- try {
- content = (String) context.configCache.get(configFile);
- } catch (ExecutionException e) {
- log.info("Failed to load config file: " + configFile, e);
- return;
- }
- // substitute tokens
- content = substituteStrWithTokens(content, tokensForSubstitution);
-
- try (OutputStream output = fs.create(remoteFile)) {
- org.apache.commons.io.IOUtils.write(content, output);
- } catch (IOException e) {
- log.info("Failed to create " + remoteFile);
- }
- }
-
- /**
- * Get initial component token map to be substituted into config values.
- * @return tokens to replace
- */
- public static Map<String, String> initCompTokensForSubstitute(
- ComponentInstance instance) {
- Map<String, String> tokens = new HashMap<>();
- tokens.put(COMPONENT_NAME, instance.getCompSpec().getName());
- tokens
- .put(COMPONENT_NAME_LC, instance.getCompSpec().getName().toLowerCase());
- tokens.put(COMPONENT_INSTANCE_NAME, instance.getCompInstanceName());
- tokens.put(CONTAINER_ID, instance.getContainer().getId().toString());
- tokens.put(COMPONENT_ID,
- String.valueOf(instance.getCompInstanceId().getId()));
- return tokens;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
deleted file mode 100644
index 0920a9c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.defaultImpl;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-
-public class DefaultClientProvider extends AbstractClientProvider {
-
- public DefaultClientProvider() {
- }
-
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException {
- // validate dest_file is not absolute
- if (Paths.get(configFile.getDestFile()).isAbsolute()) {
- throw new IllegalArgumentException(
- "Dest_file must not be absolute path: " + configFile.getDestFile());
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
deleted file mode 100644
index 868bba8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.defaultImpl;
-
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-
-public final class DefaultProviderFactory extends ProviderFactory {
- private static final ProviderFactory FACTORY = new
- DefaultProviderFactory();
-
- private DefaultProviderFactory() {}
-
- private static class Client {
- static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
- }
-
- private static class Server {
- static final ProviderService PROVIDER = new DefaultProviderService();
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return Client.PROVIDER;
- }
-
- @Override
- public ProviderService createServerProvider() {
- return Server.PROVIDER;
- }
-
- public static ProviderFactory getInstance() {
- return FACTORY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
deleted file mode 100644
index 33f8278..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.defaultImpl;
-
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-
-import java.io.IOException;
-
-public class DefaultProviderService extends AbstractProviderService {
-
- @Override
- public void processArtifact(AbstractLauncher launcher,
- ComponentInstance compInstance, SliderFileSystem fileSystem,
- Application application)
- throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
deleted file mode 100644
index d4a2254..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.docker;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-
-import java.io.IOException;
-
-public class DockerClientProvider extends AbstractClientProvider
- implements YarnServiceConstants {
-
- public DockerClientProvider() {
- super();
- }
-
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
- if (artifact == null) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
- }
- if (StringUtils.isEmpty(artifact.getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
deleted file mode 100644
index f30c002..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.docker;
-
-public interface DockerKeys {
- String PROVIDER_DOCKER = "docker";
- String DOCKER_PREFIX = "docker.";
- String DOCKER_IMAGE = DOCKER_PREFIX + "image";
- String DOCKER_NETWORK = DOCKER_PREFIX + "network";
- String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged";
- String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
-
- String DEFAULT_DOCKER_NETWORK = "bridge";
- Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
deleted file mode 100644
index 57330ab..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.docker;
-
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-
-public class DockerProviderFactory extends ProviderFactory {
- private static final ProviderFactory FACTORY = new
- DockerProviderFactory();
-
- private DockerProviderFactory() {
- }
-
- private static class Client {
- static final AbstractClientProvider PROVIDER = new DockerClientProvider();
- }
-
- private static class Server {
- static final ProviderService PROVIDER = new DockerProviderService();
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return Client.PROVIDER;
- }
-
- @Override
- public ProviderService createServerProvider() {
- return Server.PROVIDER;
- }
-
- public static ProviderFactory getInstance() {
- return FACTORY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
deleted file mode 100644
index 236ddd9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.docker;
-
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-
-public class DockerProviderService extends AbstractProviderService
- implements DockerKeys {
-
- public void processArtifact(AbstractLauncher launcher,
- ComponentInstance compInstance, SliderFileSystem fileSystem,
- Application application) throws IOException{
- launcher.setYarnDockerMode(true);
- launcher.setDockerImage(compInstance.getCompSpec().getArtifact().getId());
- launcher.setDockerNetwork(compInstance.getCompSpec().getConfiguration()
- .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
- String domain = compInstance.getComponent().getScheduler().getConfig()
- .get(RegistryConstants.KEY_DNS_DOMAIN);
- String hostname;
- if (domain == null || domain.isEmpty()) {
- hostname = MessageFormat
- .format("{0}.{1}.{2}", compInstance.getCompInstanceName(),
- application.getName(), RegistryUtils.currentUser());
- } else {
- hostname = MessageFormat
- .format("{0}.{1}.{2}.{3}", compInstance.getCompInstanceName(),
- application.getName(), RegistryUtils.currentUser(), domain);
- }
- launcher.setDockerHostname(hostname);
- launcher.setRunPrivilegedContainer(
- compInstance.getCompSpec().getRunPrivilegedContainer());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
deleted file mode 100644
index 01f7b20..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.tarball;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-
-public class TarballClientProvider extends AbstractClientProvider
- implements YarnServiceConstants {
-
- public TarballClientProvider() {
- }
-
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fs)
- throws IOException {
- if (artifact == null) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
- }
- if (StringUtils.isEmpty(artifact.getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- Path p = new Path(artifact.getId());
- if (!fs.exists(p)) {
- throw new IllegalArgumentException( "Artifact tarball does not exist "
- + artifact.getId());
- }
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException {
- // validate dest_file is not absolute
- if (Paths.get(configFile.getDestFile()).isAbsolute()) {
- throw new IllegalArgumentException(
- "Dest_file must not be absolute path: " + configFile.getDestFile());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
deleted file mode 100644
index 9d81f66..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.tarball;
-
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-
-public class TarballProviderFactory extends ProviderFactory {
- private static final ProviderFactory FACTORY = new
- TarballProviderFactory();
-
- private TarballProviderFactory() {
- }
-
- private static class Client {
- static final AbstractClientProvider PROVIDER = new TarballClientProvider();
- }
-
- private static class Server {
- static final ProviderService PROVIDER = new TarballProviderService();
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return Client.PROVIDER;
- }
-
- @Override
- public ProviderService createServerProvider() {
- return Server.PROVIDER;
- }
-
- public static ProviderFactory getInstance() {
- return FACTORY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
deleted file mode 100644
index 2403255..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider.tarball;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-
-import java.io.IOException;
-
-public class TarballProviderService extends AbstractProviderService {
-
- @Override
- public void processArtifact(AbstractLauncher launcher,
- ComponentInstance instance, SliderFileSystem fileSystem,
- Application application)
- throws IOException {
- Path artifact = new Path(instance.getCompSpec().getArtifact().getId());
- if (!fileSystem.isFile(artifact)) {
- throw new IOException(
- "Package doesn't exist as a resource: " + artifact.toString());
- }
- log.info("Adding resource {}", artifact.toString());
- LocalResourceType type = LocalResourceType.ARCHIVE;
- LocalResource packageResource = fileSystem.createAmResource(artifact, type);
- launcher.addLocalResource(APP_LIB_DIR, packageResource);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
deleted file mode 100644
index 56634f6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.registry;
-
-/**
- * These are constants unique to the Slider AM
- */
-public class CustomRegistryConstants {
-
- public static final String MANAGEMENT_REST_API =
- "classpath:org.apache.slider.management";
-
- public static final String REGISTRY_REST_API =
- "classpath:org.apache.slider.registry";
-
- public static final String PUBLISHER_REST_API =
- "classpath:org.apache.slider.publisher";
-
- public static final String PUBLISHER_CONFIGURATIONS_API =
- "classpath:org.apache.slider.publisher.configurations";
-
- public static final String PUBLISHER_EXPORTS_API =
- "classpath:org.apache.slider.publisher.exports";
-
- public static final String PUBLISHER_DOCUMENTS_API =
- "classpath:org.apache.slider.publisher.documents";
-
- public static final String AGENT_SECURE_REST_API =
- "classpath:org.apache.slider.agents.secure";
-
- public static final String AGENT_ONEWAY_REST_API =
- "classpath:org.apache.slider.agents.oneway";
-
- public static final String AM_IPC_PROTOCOL =
- "classpath:org.apache.slider.appmaster.ipc";
-
- public static final String AM_REST_BASE =
- "classpath:org.apache.slider.client.rest";
-
- public static final String WEB_UI = "http://";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
deleted file mode 100644
index ef5ed91..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.registry;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.registry.client.api.BindFlags;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
-
-/**
- * Registry view for providers. This tracks where the service
- * is registered, offers access to the record and other things.
- */
-public class YarnRegistryViewForProviders {
- private static final Log LOG =
- LogFactory.getLog(YarnRegistryViewForProviders.class);
-
- private final RegistryOperations registryOperations;
- private final String user;
- private final String sliderServiceClass;
- private final String instanceName;
- /**
- * Record used where the service registered itself.
- * Null until the service is registered
- */
- private ServiceRecord selfRegistration;
-
- /**
- * Path where record was registered
- * Null until the service is registered
- */
- private String selfRegistrationPath;
-
- public YarnRegistryViewForProviders(RegistryOperations registryOperations,
- String user,
- String sliderServiceClass,
- String instanceName,
- ApplicationAttemptId applicationAttemptId) {
- Preconditions.checkArgument(registryOperations != null,
- "null registry operations");
- Preconditions.checkArgument(user != null, "null user");
- Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
- "unset service class");
- Preconditions.checkArgument(SliderUtils.isSet(instanceName),
- "instanceName");
- Preconditions.checkArgument(applicationAttemptId != null,
- "null applicationAttemptId");
- this.registryOperations = registryOperations;
- this.user = user;
- this.sliderServiceClass = sliderServiceClass;
- this.instanceName = instanceName;
- }
-
- public String getUser() {
- return user;
- }
-
-
- private void setSelfRegistration(ServiceRecord selfRegistration) {
- this.selfRegistration = selfRegistration;
- }
-
- /**
- * Get the path to where the service has registered itself.
- * Null until the service is registered
- * @return the service registration path.
- */
- public String getSelfRegistrationPath() {
- return selfRegistrationPath;
- }
-
- /**
- * Get the absolute path to where the service has registered itself.
- * This includes the base registry path
- * Null until the service is registered
- * @return the service registration path.
- */
- public String getAbsoluteSelfRegistrationPath() {
- if (selfRegistrationPath == null) {
- return null;
- }
- String root = registryOperations.getConfig().getTrimmed(
- RegistryConstants.KEY_REGISTRY_ZK_ROOT,
- RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
- return RegistryPathUtils.join(root, selfRegistrationPath);
- }
-
- /**
- * Add a component under the slider name/entry
- * @param componentName component name
- * @param record record to put
- * @throws IOException
- */
- public void putComponent(String componentName,
- ServiceRecord record) throws
- IOException {
- putComponent(sliderServiceClass, instanceName,
- componentName,
- record);
- }
-
- /**
- * Add a component
- * @param serviceClass service class to use under ~user
- * @param componentName component name
- * @param record record to put
- * @throws IOException
- */
- public void putComponent(String serviceClass,
- String serviceName,
- String componentName,
- ServiceRecord record) throws IOException {
- String path = RegistryUtils.componentPath(
- user, serviceClass, serviceName, componentName);
- registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.bind(path, record, BindFlags.OVERWRITE);
- }
-
- /**
- * Add a service under a path, optionally purging any history
- * @param username user
- * @param serviceClass service class to use under ~user
- * @param serviceName name of the service
- * @param record service record
- * @param deleteTreeFirst perform recursive delete of the path first.
- * @return the path the service was created at
- * @throws IOException
- */
- public String putService(String username,
- String serviceClass,
- String serviceName,
- ServiceRecord record,
- boolean deleteTreeFirst) throws IOException {
- String path = RegistryUtils.servicePath(
- username, serviceClass, serviceName);
- if (deleteTreeFirst) {
- registryOperations.delete(path, true);
- }
- registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.bind(path, record, BindFlags.OVERWRITE);
- return path;
- }
-
- /**
- * Add a service under a path for the current user
- * @param record service record
- * @param deleteTreeFirst perform recursive delete of the path first
- * @return the path the service was created at
- * @throws IOException
- */
- public String registerSelf(
- ServiceRecord record,
- boolean deleteTreeFirst) throws IOException {
- selfRegistrationPath =
- putService(user, sliderServiceClass, instanceName, record, deleteTreeFirst);
- setSelfRegistration(record);
- return selfRegistrationPath;
- }
-
- /**
- * Delete a component
- * @param containerId component name
- * @throws IOException
- */
- public void deleteComponent(ComponentInstanceId instanceId,
- String containerId) throws IOException {
- String path = RegistryUtils.componentPath(
- user, sliderServiceClass, instanceName,
- containerId);
- LOG.info(instanceId + ": Deleting registry path " + path);
- registryOperations.delete(path, false);
- }
-
- /**
- * Delete the children of a path -but not the path itself.
- * It is not an error if the path does not exist
- * @param path path to delete
- * @param recursive flag to request recursive deletes
- * @throws IOException IO problems
- */
- public void deleteChildren(String path, boolean recursive) throws IOException {
- List<String> childNames = null;
- try {
- childNames = registryOperations.list(path);
- } catch (PathNotFoundException e) {
- return;
- }
- for (String childName : childNames) {
- String child = join(path, childName);
- registryOperations.delete(child, recursive);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java
deleted file mode 100644
index 2d01bef..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/BaseRestClient.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-import java.net.URI;
-
-
-/**
- * This is a base class for Jersey REST clients in Slider.
- * It supports the execution of operations —with
- * exceptions uprated to IOExceptions when needed.
- * <p>
- * Subclasses can use these operations to provide an API-like view
- * of the REST model
- */
-public class BaseRestClient {
- private static final Logger log =
- LoggerFactory.getLogger(BaseRestClient.class);
- private final Client client;
-
- public BaseRestClient(
- Client client) {
- Preconditions.checkNotNull(client, "null jersey client");
- this.client = client;
- }
-
- /**
- * Get the jersey client
- * @return jersey client
- */
- public Client getClient() {
- return client;
- }
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param c class to build
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, Class<T> c)
- throws IOException {
- try {
- Preconditions.checkArgument(c != null);
- log.debug("{}} {}", method, resource.getURI());
- return resource.accept(MediaType.APPLICATION_JSON_TYPE)
- .method(method.getVerb(), c);
- } catch (ClientHandlerException ex) {
- throw ExceptionConverter.convertJerseyException(method.getVerb(),
- resource.getURI().toString(),
- ex);
- } catch (UniformInterfaceException ex) {
- throw UgiJerseyBinding.uprateFaults(method,
- resource.getURI().toString(),
- ex);
- }
- }
-
- /**
- * Execute the operation. Failures are raised as IOException subclasses
- * @param method method to execute
- * @param resource resource to work against
- * @param t type to work with
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
- throws IOException {
- try {
- Preconditions.checkArgument(t != null);
- log.debug("{}} {}", method, resource.getURI());
- resource.accept(MediaType.APPLICATION_JSON_TYPE);
- return resource.method(method.getVerb(), t);
- } catch (ClientHandlerException ex) {
- throw ExceptionConverter.convertJerseyException(method.getVerb(),
- resource.getURI().toString(),
- ex);
- } catch (UniformInterfaceException ex) {
- throw UgiJerseyBinding.uprateFaults(method, resource.getURI().toString(),
- ex);
- }
- }
-
-
- /**
- * Execute the GET operation. Failures are raised as IOException subclasses
- * @param resource resource to work against
- * @param c class to build
- * @param <T> type expected
- * @return an instance of the type T
- * @throws IOException on any failure
- */
- public <T> T get(WebResource resource, Class<T> c) throws IOException {
- return exec(HttpVerb.GET, resource, c);
- }
-
- /**
- * Create a Web resource from the client.
- *
- * @param u the URI of the resource.
- * @return the Web resource.
- */
- public WebResource resource(URI u) {
- return client.resource(u);
- }
-
- /**
- * Create a Web resource from the client.
- *
- * @param url the URI of the resource.
- * @return the Web resource.
- */
-
- public WebResource resource(String url) {
- return client.resource(url);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java
deleted file mode 100644
index 12fdc79..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/ExceptionConverter.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import org.apache.hadoop.fs.InvalidRequestException;
-import org.apache.hadoop.fs.PathAccessDeniedException;
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.yarn.webapp.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletResponse;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-/**
- * static methods to convert exceptions into different types, including
- * extraction of details and finer-grained conversions.
- */
-public class ExceptionConverter {
- private static final Logger
- log = LoggerFactory.getLogger(ExceptionConverter.class);
-
- /**
- * Uprate error codes 400 and up into faults;
- * 404 is converted to a {@link FileNotFoundException},
- * 401 to {@link ForbiddenException}
- * FileNotFoundException for an unknown resource
- * PathAccessDeniedException for access denied
- * PathIOException for anything else
- * @param verb HTTP Verb used
- * @param targetURL URL being targeted
- * @param exception original exception
- * @return a new exception, the original one nested as a cause
- */
- public static IOException convertJerseyException(String verb,
- String targetURL,
- UniformInterfaceException exception) {
-
- IOException ioe = null;
- ClientResponse response = exception.getResponse();
- if (response != null) {
- int status = response.getStatus();
- String body = "";
- try {
- if (response.hasEntity()) {
- body = response.getEntity(String.class);
- log.error("{} {} returned status {} and body\n{}",
- verb, targetURL, status, body);
- } else {
- log.error("{} {} returned status {} and empty body",
- verb, targetURL, status);
- }
- } catch (Exception e) {
- log.warn("Failed to extract body from client response", e);
- }
-
- if (status == HttpServletResponse.SC_UNAUTHORIZED
- || status == HttpServletResponse.SC_FORBIDDEN) {
- ioe = new PathAccessDeniedException(targetURL);
- } else if (status == HttpServletResponse.SC_BAD_REQUEST
- || status == HttpServletResponse.SC_NOT_ACCEPTABLE
- || status == HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE) {
- // bad request
- ioe = new InvalidRequestException(
- String.format("Bad %s request: status code %d against %s",
- verb, status, targetURL));
- } else if (status > 400 && status < 500) {
- ioe = new FileNotFoundException(targetURL);
- }
- if (ioe == null) {
- ioe = new PathIOException(targetURL,
- verb + " " + targetURL
- + " failed with status code : " + status
- + ":" + exception);
- }
- } else {
- ioe = new PathIOException(targetURL,
- verb + " " + targetURL + " failed: " + exception);
- }
- ioe.initCause(exception);
- return ioe;
- }
-
- /**
- * Handle a client-side Jersey exception.
- * <p>
- * If there's an inner IOException, return that.
- * <p>
- * Otherwise: create a new wrapper IOE including verb and target details
- * @param verb HTTP Verb used
- * @param targetURL URL being targeted
- * @param exception original exception
- * @return an exception to throw
- */
- public static IOException convertJerseyException(String verb,
- String targetURL,
- ClientHandlerException exception) {
- if (exception.getCause() instanceof IOException) {
- return (IOException)exception.getCause();
- } else {
- IOException ioe = new IOException(
- verb + " " + targetURL + " failed: " + exception);
- ioe.initCause(exception);
- return ioe;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java
deleted file mode 100644
index 93f9082..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/HttpVerb.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-/**
- * Http verbs with details on what they support in terms of submit and
- * response bodies.
- * <p>
- * Those verbs which do support bodies in the response MAY NOT return it;
- * if the response code is 204 then the answer is "no body", but the operation
- * is considered a success.
- */
-public enum HttpVerb {
- GET("GET", false, true),
- POST("POST", true, true),
- PUT("PUT", true, true),
- DELETE("DELETE", false, true),
- HEAD("HEAD", false, false);
-
- private final String verb;
- private final boolean hasUploadBody;
- private final boolean hasResponseBody;
-
- HttpVerb(String verb, boolean hasUploadBody, boolean hasResponseBody) {
- this.verb = verb;
- this.hasUploadBody = hasUploadBody;
- this.hasResponseBody = hasResponseBody;
- }
-
- public String getVerb() {
- return verb;
- }
-
- public boolean hasUploadBody() {
- return hasUploadBody;
- }
-
- public boolean hasResponseBody() {
- return hasResponseBody;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java
deleted file mode 100644
index fcd7f55..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/SliderURLConnectionFactory.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
-import org.apache.hadoop.security.ssl.SSLFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLSocketFactory;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.net.URLConnection;
-import java.security.GeneralSecurityException;
-
-/**
- * Factory for URL connections; used behind the scenes in the Jersey integration.
- * <p>
- * Derived from the WebHDFS implementation.
- */
-public class SliderURLConnectionFactory {
- private static final Logger log =
- LoggerFactory.getLogger(SliderURLConnectionFactory.class);
-
- /**
- * Timeout for socket connects and reads
- */
- public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute
- private final ConnectionConfigurator connConfigurator;
-
- private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator();
-
- /**
- * Construct a new URLConnectionFactory based on the configuration. It will
- * try to load SSL certificates when it is specified.
- */
- public static SliderURLConnectionFactory newInstance(Configuration conf) {
- ConnectionConfigurator conn;
- try {
- conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
- } catch (Exception e) {
- log.debug("Cannot load customized SSL configuration.", e);
- conn = DEFAULT_CONFIGURATOR;
- }
- return new SliderURLConnectionFactory(conn);
- }
-
- private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) {
- this.connConfigurator = connConfigurator;
- }
-
- /**
- * Create a new ConnectionConfigurator for SSL connections
- */
- private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
- Configuration conf) throws IOException, GeneralSecurityException {
- final SSLFactory factory;
- final SSLSocketFactory sf;
- final HostnameVerifier hv;
-
- factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
- factory.init();
- sf = factory.createSSLSocketFactory();
- hv = factory.getHostnameVerifier();
-
- return new ConnectionConfigurator() {
- @Override
- public HttpURLConnection configure(HttpURLConnection conn)
- throws IOException {
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection c = (HttpsURLConnection) conn;
- c.setSSLSocketFactory(sf);
- c.setHostnameVerifier(hv);
- }
- SliderURLConnectionFactory.setupConnection(conn, timeout);
- return conn;
- }
- };
- }
-
- /**
- * Opens a url with read and connect timeouts
- *
- * @param url
- * to open
- * @return URLConnection
- * @throws IOException
- */
- public URLConnection openConnection(URL url) throws IOException {
- try {
- return openConnection(url, false);
- } catch (AuthenticationException e) {
- // Unreachable
- return null;
- }
- }
-
- /**
- * Opens a url with read and connect timeouts
- *
- * @param url
- * URL to open
- * @param isSpnego
- * whether the url should be authenticated via SPNEGO
- * @return URLConnection
- * @throws IOException
- * @throws AuthenticationException
- */
- public URLConnection openConnection(URL url, boolean isSpnego)
- throws IOException, AuthenticationException {
- if (isSpnego) {
- log.debug("open AuthenticatedURL connection {}", url);
- UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
- final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
- return new AuthenticatedURL(new KerberosUgiAuthenticator(),
- connConfigurator).openConnection(url, authToken);
- } else {
- log.debug("open URL connection {}", url);
- URLConnection connection = url.openConnection();
- if (connection instanceof HttpURLConnection) {
- connConfigurator.configure((HttpURLConnection) connection);
- }
- return connection;
- }
- }
-
- /**
- * Sets connection parameters on the given URLConnection
- *
- * @param connection
- * URLConnection to set
- * @param socketTimeout
- * the connection and read timeout of the connection.
- */
- private static void setupConnection(URLConnection connection, int socketTimeout) {
- connection.setConnectTimeout(socketTimeout);
- connection.setReadTimeout(socketTimeout);
- connection.setUseCaches(false);
- if (connection instanceof HttpURLConnection) {
- ((HttpURLConnection) connection).setInstanceFollowRedirects(true);
- }
- }
-
- private static class BasicConfigurator implements ConnectionConfigurator {
- @Override
- public HttpURLConnection configure(HttpURLConnection conn)
- throws IOException {
- SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT);
- return conn;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
deleted file mode 100644
index 7baa284..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Build a single command line to include in the container commands;
- * Special support for JVM command buildup.
- */
-public class CommandLineBuilder {
- protected final List<String> argumentList = new ArrayList<>(20);
-
- /**
- * Add an entry to the command list
- * @param args arguments -these will be converted strings
- */
- public void add(Object... args) {
- for (Object arg : args) {
- argumentList.add(arg.toString());
- }
- }
-
- // Get the number of arguments
- public int size() {
- return argumentList.size();
- }
-
- /**
- * Append the output and error files to the tail of the command
- * @param stdout out
- * @param stderr error. Set this to null to append into stdout
- */
- public void addOutAndErrFiles(String stdout, String stderr) {
- Preconditions.checkNotNull(stdout, "Null output file");
- Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
- // write out the path output
- argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stdout);
- if (stderr != null) {
- argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stderr);
- } else {
- argumentList.add("2>&1");
- }
- }
-
- /**
- * This just returns the command line
- * @see #build()
- * @return the command line
- */
- @Override
- public String toString() {
- return build();
- }
-
- /**
- * Build the command line
- * @return the command line
- */
- public String build() {
- return SliderUtils.join(argumentList, " ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
deleted file mode 100644
index fcbb69b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-public class ContainerLaunchService extends AbstractService{
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(ContainerLaunchService.class);
-
- private ExecutorService executorService;
- private SliderFileSystem fs;
-
- public ContainerLaunchService(SliderFileSystem fs) {
- super(ContainerLaunchService.class.getName());
- this.fs = fs;
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- executorService = Executors.newCachedThreadPool();
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (executorService != null) {
- executorService.shutdownNow();
- }
- super.serviceStop();
- }
-
- public void launchCompInstance(Application application,
- ComponentInstance instance, Container container) {
- ContainerLauncher launcher =
- new ContainerLauncher(application, instance, container);
- executorService.execute(launcher);
- }
-
- private class ContainerLauncher implements Runnable {
- public final Container container;
- public final Application application;
- public ComponentInstance instance;
-
- public ContainerLauncher(
- Application application,
- ComponentInstance instance, Container container) {
- this.container = container;
- this.application = application;
- this.instance = instance;
- }
-
- @Override public void run() {
- Component compSpec = instance.getCompSpec();
- ProviderService provider = ProviderFactory.getProviderService(
- compSpec.getArtifact());
- AbstractLauncher launcher = new AbstractLauncher(fs, null);
- try {
- provider.buildContainerLaunchContext(launcher, application,
- instance, fs, getConfig());
- instance.getComponent().getScheduler().getNmClient()
- .startContainerAsync(container,
- launcher.completeContainerLaunch());
- } catch (Exception e) {
- LOG.error(instance.getCompInstanceId()
- + ": Failed to launch container. ", e);
-
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
deleted file mode 100644
index fce58e5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CredentialUtils.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-
-/**
- * Utils to work with credentials and tokens.
- *
- * Designed to be movable to Hadoop core
- */
-public final class CredentialUtils {
-
- private CredentialUtils() {
- }
-
- private static final Logger LOG =
- LoggerFactory.getLogger(CredentialUtils.class);
-
- /**
- * Save credentials to a byte buffer. Returns null if there were no
- * credentials to save
- * @param credentials credential set
- * @return a byte buffer of serialized tokens
- * @throws IOException if the credentials could not be written to the stream
- */
- public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException {
- ByteBuffer buffer = null;
- if (!credentials.getAllTokens().isEmpty()) {
- DataOutputBuffer dob = new DataOutputBuffer();
- try {
- credentials.writeTokenStorageToStream(dob);
- } finally {
- dob.close();
- }
- buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- }
- return buffer;
- }
-
- /**
- * Save credentials to a file
- * @param file file to save to (will be overwritten)
- * @param credentials credentials to write
- * @throws IOException
- */
- public static void saveTokens(File file,
- Credentials credentials) throws IOException {
- try(DataOutputStream daos = new DataOutputStream(
- new FileOutputStream(file))) {
- credentials.writeTokenStorageToStream(daos);
- }
- }
-
- /**
- * Look up and return the resource manager's principal. This method
- * automatically does the <code>_HOST</code> replacement in the principal and
- * correctly handles HA resource manager configurations.
- *
- * From: YARN-4629
- * @param conf the {@link Configuration} file from which to read the
- * principal
- * @return the resource manager's principal string
- * @throws IOException thrown if there's an error replacing the host name
- */
- public static String getRMPrincipal(Configuration conf) throws IOException {
- String principal = conf.get(RM_PRINCIPAL, "");
- String hostname;
- Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);
-
- if (HAUtil.isHAEnabled(conf)) {
- YarnConfiguration yarnConf = new YarnConfiguration(conf);
- if (yarnConf.get(RM_HA_ID) == null) {
- // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
- // Any valid RM HA ID should work.
- String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
- Preconditions.checkState((rmIds != null) && (rmIds.length > 0),
- "Not set " + RM_HA_IDS);
- yarnConf.set(RM_HA_ID, rmIds[0]);
- }
-
- hostname = yarnConf.getSocketAddr(
- RM_ADDRESS,
- DEFAULT_RM_ADDRESS,
- DEFAULT_RM_PORT).getHostName();
- } else {
- hostname = conf.getSocketAddr(
- RM_ADDRESS,
- DEFAULT_RM_ADDRESS,
- DEFAULT_RM_PORT).getHostName();
- }
- return SecurityUtil.getServerPrincipal(principal, hostname);
- }
-
- /**
- * Create and add any filesystem delegation tokens with
- * the RM(s) configured to be able to renew them. Returns null
- * on an insecure cluster (i.e. harmless)
- * @param conf configuration
- * @param fs filesystem
- * @param credentials credentials to update
- * @return a list of all added tokens.
- * @throws IOException
- */
- public static Token<?>[] addRMRenewableFSDelegationTokens(Configuration conf,
- FileSystem fs,
- Credentials credentials) throws IOException {
- Preconditions.checkArgument(conf != null);
- Preconditions.checkArgument(credentials != null);
- if (UserGroupInformation.isSecurityEnabled()) {
- return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf),
- credentials);
- }
- return null;
- }
-
- /**
- * Add an FS delegation token which can be renewed by the current user
- * @param fs filesystem
- * @param credentials credentials to update
- * @throws IOException problems.
- */
- public static void addSelfRenewableFSDelegationTokens(
- FileSystem fs,
- Credentials credentials) throws IOException {
- Preconditions.checkArgument(fs != null);
- Preconditions.checkArgument(credentials != null);
- fs.addDelegationTokens(
- getSelfRenewer(),
- credentials);
- }
-
- public static String getSelfRenewer() throws IOException {
- return UserGroupInformation.getLoginUser().getShortUserName();
- }
-
- /**
- * Create and add an RM delegation token to the credentials
- * @param yarnClient Yarn Client
- * @param credentials to add token to
- * @return the token which was added
- * @throws IOException
- * @throws YarnException
- */
- public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient,
- Credentials credentials)
- throws IOException, YarnException {
- Configuration conf = yarnClient.getConfig();
- Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf));
- Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
- Token<TokenIdentifier> rmDelegationToken =
- ConverterUtils.convertFromYarn(
- yarnClient.getRMDelegationToken(rmPrincipal),
- rmDTService);
- credentials.addToken(rmDelegationToken.getService(), rmDelegationToken);
- return rmDelegationToken;
- }
-
- public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(
- Configuration conf,
- Credentials credentials)
- throws IOException, YarnException {
- if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
- LOG.debug("Timeline service enabled -fetching token");
-
- try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
- timelineClient.init(conf);
- timelineClient.start();
- Token<TimelineDelegationTokenIdentifier> token =
- timelineClient.getDelegationToken(
- CredentialUtils.getRMPrincipal(conf));
- credentials.addToken(token.getService(), token);
- return token;
- }
- } else {
- LOG.debug("Timeline service is disabled");
- return null;
- }
- }
-
- /**
- * Filter a list of tokens from a set of credentials
- * @param credentials credential source (a new credential set os re
- * @param filter List of tokens to strip out
- * @return a new, filtered, set of credentials
- */
- public static Credentials filterTokens(Credentials credentials,
- List<Text> filter) {
- Credentials result = new Credentials(credentials);
- Iterator<Token<? extends TokenIdentifier>> iter =
- result.getAllTokens().iterator();
- while (iter.hasNext()) {
- Token<? extends TokenIdentifier> token = iter.next();
- LOG.debug("Token {}", token.getKind());
- if (filter.contains(token.getKind())) {
- LOG.debug("Filtering token {}", token.getKind());
- iter.remove();
- }
- }
- return result;
- }
-
- public static String dumpTokens(Credentials credentials, String separator) {
- ArrayList<Token<? extends TokenIdentifier>> sorted =
- new ArrayList<>(credentials.getAllTokens());
- Collections.sort(sorted, new TokenComparator());
- StringBuilder buffer = new StringBuilder(sorted.size()* 128);
- for (Token<? extends TokenIdentifier> token : sorted) {
- buffer.append(tokenToString(token)).append(separator);
- }
- return buffer.toString();
- }
-
- /**
- * Create a string for people to look at
- * @param token token to convert to a string form
- * @return a printable view of the token
- */
- public static String tokenToString(Token<? extends TokenIdentifier> token) {
- DateFormat df = DateFormat.getDateTimeInstance(
- DateFormat.SHORT, DateFormat.SHORT);
- StringBuilder buffer = new StringBuilder(128);
- buffer.append(token.toString());
- try {
- TokenIdentifier ti = token.decodeIdentifier();
- buffer.append("; ").append(ti);
- if (ti instanceof AbstractDelegationTokenIdentifier) {
- // details in human readable form, and compensate for information HDFS DT omits
- AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti;
- buffer.append("; Renewer: ").append(dt.getRenewer());
- buffer.append("; Issued: ")
- .append(df.format(new Date(dt.getIssueDate())));
- buffer.append("; Max Date: ")
- .append(df.format(new Date(dt.getMaxDate())));
- }
- } catch (IOException e) {
- //marshall problem; not ours
- LOG.debug("Failed to decode {}: {}", token, e, e);
- }
- return buffer.toString();
- }
-
- /**
- * Get the expiry time of a token.
- * @param token token to examine
- * @return the time in milliseconds after which the token is invalid.
- * @throws IOException
- */
- public static long getTokenExpiryTime(Token token) throws IOException {
- TokenIdentifier identifier = token.decodeIdentifier();
- Preconditions.checkState(identifier instanceof AbstractDelegationTokenIdentifier,
- "Token %s of type: %s has an identifier which cannot be examined: %s",
- token, token.getClass(), identifier);
- AbstractDelegationTokenIdentifier id =
- (AbstractDelegationTokenIdentifier) identifier;
- return id.getMaxDate();
- }
-
- private static class TokenComparator
- implements Comparator<Token<? extends TokenIdentifier>>, Serializable {
- @Override
- public int compare(Token<? extends TokenIdentifier> left,
- Token<? extends TokenIdentifier> right) {
- return left.getKind().toString().compareTo(right.getKind().toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
deleted file mode 100644
index cbcb0d6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-
-import java.util.Map;
-
-/**
- * Command line builder purely for the Java CLI.
- * Some of the <code>define</code> methods are designed to work with Hadoop tool and
- * Slider launcher applications.
- */
-public class JavaCommandLineBuilder extends CommandLineBuilder {
-
- public JavaCommandLineBuilder() {
- add(getJavaBinary());
- }
-
- /**
- * Get the java binary. This is called in the constructor so don't try and
- * do anything other than return a constant.
- * @return the path to the Java binary
- */
- protected String getJavaBinary() {
- return ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java";
- }
-
- /**
- * Set the size of the heap if a non-empty heap is passed in.
- * @param heap empty string or something like "128M" ,"1G" etc. The value is
- * trimmed.
- */
- public void setJVMHeap(String heap) {
- if (SliderUtils.isSet(heap)) {
- add("-Xmx" + heap.trim());
- }
- }
-
- /**
- * Turn Java assertions on
- */
- public void enableJavaAssertions() {
- add("-ea");
- add("-esa");
- }
-
- /**
- * Add a system property definition -must be used before setting the main entry point
- * @param property
- * @param value
- */
- public void sysprop(String property, String value) {
- Preconditions.checkArgument(property != null, "null property name");
- Preconditions.checkArgument(value != null, "null value");
- add("-D" + property + "=" + value);
- }
-
- public JavaCommandLineBuilder forceIPv4() {
- sysprop("java.net.preferIPv4Stack", "true");
- return this;
- }
-
- public JavaCommandLineBuilder headless() {
- sysprop("java.awt.headless", "true");
- return this;
- }
-
- public boolean addConfOption(Configuration conf, String key) {
- return defineIfSet(key, conf.get(key));
- }
-
- /**
- * Add a varargs list of configuration parameters —if they are present
- * @param conf configuration source
- * @param keys keys
- */
- public void addConfOptions(Configuration conf, String... keys) {
- for (String key : keys) {
- addConfOption(conf, key);
- }
- }
-
- /**
- * Add all configuration options which match the prefix
- * @param conf configuration
- * @param prefix prefix, e.g {@code "slider."}
- * @return the number of entries copied
- */
- public int addPrefixedConfOptions(Configuration conf, String prefix) {
- int copied = 0;
- for (Map.Entry<String, String> entry : conf) {
- if (entry.getKey().startsWith(prefix)) {
- define(entry.getKey(), entry.getValue());
- copied++;
- }
- }
- return copied;
- }
-
- /**
- * Ass a configuration option to the command line of the application
- * @param conf configuration
- * @param key key
- * @param defVal default value
- * @return the resolved configuration option
- * @throws IllegalArgumentException if key is null or the looked up value
- * is null (that is: the argument is missing and devVal was null.
- */
- public String addConfOptionToCLI(Configuration conf,
- String key,
- String defVal) {
- Preconditions.checkArgument(key != null, "null key");
- String val = conf.get(key, defVal);
- define(key, val);
- return val;
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
- * @param key key
- * @param val value
- * @throws IllegalArgumentException if either argument is null
- */
- public void define(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- Preconditions.checkArgument(val != null, "null value");
- add("-D", key + "=" + val);
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
- * is not null
- * @param key key
- * @param val value
- */
- public boolean defineIfSet(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- if (val != null) {
- define(key, val);
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Add a mandatory config option
- * @param conf configuration
- * @param key key
- * @throws BadConfigException if the key is missing
- */
- public void addMandatoryConfOption(Configuration conf,
- String key) throws BadConfigException {
- if (!addConfOption(conf, key)) {
- throw new BadConfigException("Missing configuration option: " + key);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
deleted file mode 100644
index db9de7a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadClusterStateException.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-
-/**
- * The system is in a bad state
- */
-public class BadClusterStateException extends SliderException {
- public BadClusterStateException(String message,
- Object... args) {
- super(EXIT_BAD_STATE, message, args);
- }
-
- public BadClusterStateException(Throwable throwable,
- String message, Object... args) {
- super(EXIT_BAD_STATE, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
deleted file mode 100644
index 41e3251..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadCommandArgumentsException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-public class BadCommandArgumentsException extends SliderException {
- public BadCommandArgumentsException(String s, Object... args) {
- super(EXIT_COMMAND_ARGUMENT_ERROR, s, args);
- }
-
- public BadCommandArgumentsException(Throwable throwable, String message,
- Object... args) {
- super(EXIT_COMMAND_ARGUMENT_ERROR, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
deleted file mode 100644
index 8199c3c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/BadConfigException.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-/**
- * An exception to raise on a bad configuration
- */
-public class BadConfigException extends SliderException {
-
- public BadConfigException(String s) {
- super(EXIT_BAD_CONFIGURATION, s);
- }
-
- public BadConfigException(String message, Object... args) {
- super(EXIT_BAD_CONFIGURATION, message, args);
- }
-
- public BadConfigException(
- Throwable throwable,
- String message, Object... args) {
- super(EXIT_BAD_CONFIGURATION, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
deleted file mode 100644
index 3577b59..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ErrorStrings.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-public interface ErrorStrings {
- String E_UNSTABLE_CLUSTER = "Unstable Application Instance :";
- String E_CLUSTER_RUNNING = "Application Instance running";
- String E_ALREADY_EXISTS = "already exists";
- String PRINTF_E_INSTANCE_ALREADY_EXISTS = "Application Instance \"%s\" already exists and is defined in %s";
- String PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS = "Application Instance dir already exists: %s";
- String E_MISSING_PATH = "Missing path ";
- String E_INCOMPLETE_CLUSTER_SPEC =
- "Cluster specification is marked as incomplete: ";
- String E_UNKNOWN_INSTANCE = "Unknown application instance ";
- String E_DESTROY_CREATE_RACE_CONDITION =
- "created while it was being destroyed";
- String E_UNKNOWN_ROLE = "Unknown role ";
- /**
- * ERROR Strings
- */
- String ERROR_NO_ACTION = "No action specified";
- String ERROR_UNKNOWN_ACTION = "Unknown command: ";
- String ERROR_NOT_ENOUGH_ARGUMENTS =
- "Not enough arguments for action: ";
- String ERROR_PARSE_FAILURE =
- "Failed to parse ";
- /**
- * All the remaining values after argument processing
- */
- String ERROR_TOO_MANY_ARGUMENTS =
- "Too many arguments";
- String ERROR_DUPLICATE_ENTRY = "Duplicate entry for ";
- String E_APPLICATION_NOT_RUNNING = "Application not running";
- String E_FINISHED_APPLICATION = E_APPLICATION_NOT_RUNNING + ": %s state=%s ";
- String E_NO_IMAGE_OR_HOME_DIR_SPECIFIED =
- "Neither an image path nor binary home directory were specified";
- String E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED =
- "Both application image path and home dir have been provided";
- String E_CONFIGURATION_DIRECTORY_NOT_FOUND =
- "Configuration directory \"%s\" not found";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
deleted file mode 100644
index d66b860..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ExitCodeProvider.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-/**
- * Get the exit code of an exception. Making it an interface allows
- * us to retrofit exit codes onto existing classes
- */
-public interface ExitCodeProvider {
-
- /**
- * Method to get the exit code
- * @return the exit code
- */
- int getExitCode();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
deleted file mode 100644
index 9657536..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/LauncherExitCodes.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-/*
- * Common Exit codes
- * <p>
- * Exit codes from 64 up are application specific.
- * <p>
- * Many of the exit codes are designed to resemble HTTP error codes,
- * squashed into a single byte. e.g 44 , "not found" is the equivalent
- * of 404
- * <pre>
- * 0-10: general command issues
- * 30-39: equivalent to the 3XX responses, where those responses are
- * considered errors by the application.
- * 40-49: request-related errors
- * 50-59: server-side problems. These may be triggered by the request.
- * 64- : application specific error codes
- * </pre>
- */
-public interface LauncherExitCodes {
-
- /**
- * 0: success
- */
- int EXIT_SUCCESS = 0;
-
- /**
- * -1: generic "false" response. The operation worked but
- * the result was not true
- */
- int EXIT_FALSE = -1;
-
- /**
- * Exit code when a client requested service termination: {@value}
- */
- int EXIT_CLIENT_INITIATED_SHUTDOWN = 1;
-
- /**
- * Exit code when targets could not be launched: {@value}
- */
- int EXIT_TASK_LAUNCH_FAILURE = 2;
-
- /**
- * Exit code when a control-C, kill -3, signal was picked up: {@value}
- */
- int EXIT_INTERRUPTED = 3;
-
- /**
- * Exit code when a usage message was printed: {@value}
- */
- int EXIT_USAGE = 4;
-
- /**
- * Exit code when something happened but we can't be specific: {@value}
- */
- int EXIT_OTHER_FAILURE = 5;
-
- /**
- * Exit code on connectivity problems: {@value}
- */
- int EXIT_MOVED = 31;
-
- /**
- * found: {@value}.
- * <p>
- * This is low value as in HTTP it is normally a success/redirect;
- * whereas on the command line 0 is the sole success code.
- * <p>
- * <code>302 Found</code>
- */
- int EXIT_FOUND = 32;
-
- /**
- * Exit code on a request where the destination has not changed
- * and (somehow) the command specified that this is an error.
- * That is, this exit code is somehow different from a "success"
- * : {@value}
- * <p>
- * <code>304 Not Modified </code>
- */
- int EXIT_NOT_MODIFIED = 34;
-
- /**
- * Exit code when the command line doesn't parse: {@value}, or
- * when it is otherwise invalid.
- * <p>
- * <code>400 BAD REQUEST</code>
- */
- int EXIT_COMMAND_ARGUMENT_ERROR = 40;
-
- /**
- * The request requires user authentication: {@value}
- * <p>
- * <code>401 Unauthorized</code>
- */
- int EXIT_UNAUTHORIZED = 41;
-
- /**
- * Forbidden action: {@value}
- * <p>
- * <code>403: Forbidden</code>
- */
- int EXIT_FORBIDDEN = 43;
-
- /**
- * Something was not found: {@value}
- * <p>
- * <code>404: NOT FOUND</code>
- */
- int EXIT_NOT_FOUND = 44;
-
- /**
- * The operation is not allowed: {@value}
- * <p>
- * <code>405: NOT ALLOWED</code>
- */
- int EXIT_OPERATION_NOT_ALLOWED = 45;
-
- /**
- * The command is somehow not acceptable: {@value}
- * <p>
- * <code>406: NOT ACCEPTABLE</code>
- */
- int EXIT_NOT_ACCEPTABLE = 46;
-
- /**
- * Exit code on connectivity problems: {@value}
- * <p>
- * <code>408: Request Timeout</code>
- */
- int EXIT_CONNECTIVITY_PROBLEM = 48;
-
- /**
- * The request could not be completed due to a conflict with the current
- * state of the resource. {@value}
- * <p>
- * <code>409: conflict</code>
- */
- int EXIT_CONFLICT = 49;
-
- /**
- * internal error: {@value}
- * <p>
- * <code>500 Internal Server Error</code>
- */
- int EXIT_INTERNAL_ERROR = 50;
-
- /**
- * Unimplemented feature: {@value}
- * <p>
- * <code>501: Not Implemented</code>
- */
- int EXIT_UNIMPLEMENTED = 51;
-
- /**
- * Service Unavailable; it may be available later: {@value}
- * <p>
- * <code>503 Service Unavailable</code>
- */
- int EXIT_SERVICE_UNAVAILABLE = 53;
-
- /**
- * The application does not support, or refuses to support this version: {@value}.
- * If raised, this is expected to be raised server-side and likely due
- * to client/server version incompatibilities.
- * <p>
- * <code> 505: Version Not Supported</code>
- */
- int EXIT_UNSUPPORTED_VERSION = 55;
-
- /**
- * Exit code when an exception was thrown from the service: {@value}
- * <p>
- * <code>5XX</code>
- */
- int EXIT_EXCEPTION_THROWN = 56;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
deleted file mode 100644
index 7be23f3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-public interface RestApiErrorMessages {
- String ERROR_APPLICATION_NAME_INVALID =
- "Application name is either empty or not provided";
- String ERROR_APPLICATION_NAME_INVALID_FORMAT =
- "Application name %s is not valid - only lower case letters, digits, " +
- "underscore and hyphen are allowed, and the name must be no more " +
- "than 63 characters";
- String ERROR_COMPONENT_NAME_INVALID =
- "Component name must be no more than %s characters: %s";
- String ERROR_USER_NAME_INVALID =
- "User name must be no more than 63 characters";
-
- String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
- String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";
- String ERROR_APPLICATION_IN_USE = "Application already exists in started"
- + " state";
- String ERROR_APPLICATION_INSTANCE_EXISTS = "Application already exists in"
- + " stopped/failed state (either restart with PUT or destroy with DELETE"
- + " before creating a new one)";
-
- String ERROR_SUFFIX_FOR_COMPONENT =
- " for component %s (nor at the global level)";
- String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
- String ERROR_ARTIFACT_FOR_COMP_INVALID =
- ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_ARTIFACT_ID_INVALID =
- "Artifact id (like docker image name) is either empty or not provided";
- String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
- ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
-
- String ERROR_RESOURCE_INVALID = "Resource is not provided";
- String ERROR_RESOURCE_FOR_COMP_INVALID =
- ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_RESOURCE_MEMORY_INVALID =
- "Application resource or memory not provided";
- String ERROR_RESOURCE_CPUS_INVALID =
- "Application resource or cpus not provided";
- String ERROR_RESOURCE_CPUS_INVALID_RANGE =
- "Unacceptable no of cpus specified, either zero or negative";
- String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
- ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
- ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
- ERROR_RESOURCE_CPUS_INVALID_RANGE
- + " for component %s (or at the global level)";
- String ERROR_CONTAINERS_COUNT_INVALID =
- "Invalid no of containers specified";
- String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
- ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
- String ERROR_DEPENDENCY_INVALID = "Dependency %s for component %s is " +
- "invalid, does not exist as a component";
- String ERROR_DEPENDENCY_CYCLE = "Invalid dependencies, a cycle may " +
- "exist: %s";
-
- String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
- "Cannot specify" + " cpus/memory along with profile";
- String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
- ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
- + " for component %s";
- String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
- "Resource profile is not " + "supported yet. Please specify cpus/memory.";
-
- String ERROR_NULL_ARTIFACT_ID =
- "Artifact Id can not be null if artifact type is none";
- String ERROR_ABSENT_NUM_OF_INSTANCE =
- "Num of instances should appear either globally or per component";
- String ERROR_ABSENT_LAUNCH_COMMAND =
- "Launch_command is required when type is not DOCKER";
-
- String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
- + " component level, needs corresponding values set at application level";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
deleted file mode 100644
index e83ccbe..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/ServiceLaunchException.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-
-import org.apache.hadoop.yarn.exceptions.YarnException;
-
-/**
- * A service launch exception that includes an exit code;
- * when caught by the ServiceLauncher, it will convert that
- * into a process exit code.
- */
-public class ServiceLaunchException extends YarnException
- implements ExitCodeProvider, LauncherExitCodes {
-
- private final int exitCode;
-
- /**
- * Create an exception with the specific exit code
- * @param exitCode exit code
- * @param cause cause of the exception
- */
- public ServiceLaunchException(int exitCode, Throwable cause) {
- super(cause);
- this.exitCode = exitCode;
- }
-
- /**
- * Create an exception with the specific exit code and text
- * @param exitCode exit code
- * @param message message to use in exception
- */
- public ServiceLaunchException(int exitCode, String message) {
- super(message);
- this.exitCode = exitCode;
- }
-
- /**
- * Create an exception with the specific exit code, text and cause
- * @param exitCode exit code
- * @param message message to use in exception
- * @param cause cause of the exception
- */
- public ServiceLaunchException(int exitCode, String message, Throwable cause) {
- super(message, cause);
- this.exitCode = exitCode;
- }
-
- /**
- * Get the exit code
- * @return the exit code
- */
- @Override
- public int getExitCode() {
- return exitCode;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
deleted file mode 100644
index 5b74b80..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/SliderException.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-
-public class SliderException extends ServiceLaunchException implements
- SliderExitCodes {
- public SliderException() {
- super(EXIT_EXCEPTION_THROWN, "SliderException");
- }
-
- public SliderException(int code, String message) {
- super(code, message);
- }
-
- public SliderException(String s) {
- super(EXIT_EXCEPTION_THROWN, s);
- }
-
- public SliderException(String s, Throwable throwable) {
- super(EXIT_EXCEPTION_THROWN, s, throwable);
- }
-
- /**
- * Format the exception as you create it
- * @param code exit code
- * @param message exception message -sprintf formatted
- * @param args arguments for the formatting
- */
- public SliderException(int code, String message, Object... args) {
- super(code, String.format(message, args));
- }
-
- /**
- * Format the exception, include a throwable.
- * The throwable comes before the message so that it is out of the varargs
- * @param code exit code
- * @param throwable thrown
- * @param message message
- * @param args arguments
- */
- public SliderException(int code,
- Throwable throwable,
- String message,
- Object... args) {
- super(code, String.format(message, args), throwable);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
deleted file mode 100644
index 3a9fa25..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/UsageException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.exceptions;
-
-/**
- * Used to raise a usage exception ... this has the exit code
- * {@link #EXIT_USAGE}
- */
-public class UsageException extends SliderException {
- public UsageException(String s, Object... args) {
- super(EXIT_USAGE, s, args);
- }
-
- public UsageException(Throwable throwable, String message,
- Object... args) {
- super(EXIT_USAGE, throwable, message, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
deleted file mode 100644
index 33e33a6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.impl.pb.client;
-
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.RPCUtil;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.service.impl.pb.service.ClientAMProtocolPB;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-
-public class ClientAMProtocolPBClientImpl
- implements ClientAMProtocol, Closeable {
-
- private ClientAMProtocolPB proxy;
-
- public ClientAMProtocolPBClientImpl(long clientVersion,
- InetSocketAddress addr, Configuration conf) throws IOException {
- RPC.setProtocolEngine(conf, ClientAMProtocolPB.class,
- ProtobufRpcEngine.class);
- proxy = RPC.getProxy(ClientAMProtocolPB.class, clientVersion, addr, conf);
-
- }
-
- @Override public FlexComponentsResponseProto flexComponents(
- FlexComponentsRequestProto request) throws IOException, YarnException {
- try {
- return proxy.flexComponents(null, request);
- } catch (ServiceException e) {
- RPCUtil.unwrapAndThrowException(e);
- }
- return null;
- }
-
- @Override
- public GetStatusResponseProto getStatus(GetStatusRequestProto request)
- throws IOException, YarnException {
- try {
- return proxy.getStatus(null, request);
- } catch (ServiceException e) {
- RPCUtil.unwrapAndThrowException(e);
- }
- return null;
- }
-
- @Override
- public StopResponseProto stop(StopRequestProto requestProto)
- throws IOException, YarnException {
- try {
- return proxy.stop(null, requestProto);
- } catch (ServiceException e) {
- RPCUtil.unwrapAndThrowException(e);
- }
- return null;
- }
-
- @Override public void close() {
- if (this.proxy != null) {
- RPC.stopProxy(this.proxy);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
deleted file mode 100644
index 6a9cd37..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.impl.pb.service;
-
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol;
-
-@ProtocolInfo(
- protocolName = "org.apache.hadoop.yarn.service.ClientAMProtocol",
- protocolVersion = 1)
-public interface ClientAMProtocolPB extends
- ClientAMProtocol.ClientAMProtocolService.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
deleted file mode 100644
index 7100781..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.impl.pb.service;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-
-import java.io.IOException;
-
-public class ClientAMProtocolPBServiceImpl implements ClientAMProtocolPB {
-
- private ClientAMProtocol real;
-
- public ClientAMProtocolPBServiceImpl(ClientAMProtocol impl) {
- this.real = impl;
- }
-
- @Override
- public FlexComponentsResponseProto flexComponents(RpcController controller,
- FlexComponentsRequestProto request) throws ServiceException {
- try {
- return real.flexComponents(request);
- } catch (IOException | YarnException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override public GetStatusResponseProto getStatus(RpcController controller,
- GetStatusRequestProto request) throws ServiceException {
- try {
- return real.getStatus(request);
- } catch (IOException | YarnException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto stop(
- RpcController controller,
- org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto request)
- throws ServiceException {
- try {
- return real.stop(request);
- } catch (IOException | YarnException e) {
- throw new ServiceException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
deleted file mode 100644
index bfe3bc9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.metrics;
-
-import org.apache.hadoop.metrics2.MetricsCollector;
-import org.apache.hadoop.metrics2.MetricsInfo;
-import org.apache.hadoop.metrics2.MetricsSource;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-
-import static org.apache.hadoop.metrics2.lib.Interns.info;
-
-@Metrics(context = "yarn-native-service")
-public class ServiceMetrics implements MetricsSource {
-
- @Metric("containers requested")
- public MutableGaugeInt containersRequested;
-
- @Metric("anti-affinity containers pending")
- public MutableGaugeInt pendingAAContainers;
-
- @Metric("containers running")
- public MutableGaugeInt containersRunning;
-
- @Metric("containers ready")
- public MutableGaugeInt containersReady;
-
- @Metric("containers desired")
- public MutableGaugeInt containersDesired;
-
- @Metric("containers succeeded")
- public MutableGaugeInt containersSucceeded;
-
- @Metric("containers failed")
- public MutableGaugeInt containersFailed;
-
- @Metric("containers preempted")
- public MutableGaugeInt containersPreempted;
-
- @Metric("containers exceeded limits")
- public MutableGaugeInt containersLimitsExceeded;
-
- @Metric("containers surplus")
- public MutableGaugeInt surplusContainers;
-
- @Metric("containers failed due to disk failure")
- public MutableGaugeInt containersDiskFailure;
-
- protected final MetricsRegistry registry;
-
- public ServiceMetrics(MetricsInfo metricsInfo) {
- registry = new MetricsRegistry(metricsInfo);
- }
-
- @Override
- public void getMetrics(MetricsCollector collector, boolean all) {
- registry.snapshot(collector.addRecord(registry.info()), all);
- }
-
- public static ServiceMetrics register(String name, String description) {
- ServiceMetrics metrics = new ServiceMetrics(info(name, description));
- DefaultMetricsSystem.instance().register(name, description, metrics);
- return metrics;
- }
-
- public void tag(String name, String description, String value) {
- registry.tag(name, description, value);
- }
-
- @Override public String toString() {
- return "ServiceMetrics{"
- + "containersRequested=" + containersRequested.value()
- + ", pendingAAContainers=" + pendingAAContainers.value()
- + ", containersRunning=" + containersRunning.value()
- + ", containersDesired=" + containersDesired.value()
- + ", containersSucceeded=" + containersSucceeded.value()
- + ", containersFailed=" + containersFailed.value()
- + ", containersPreempted=" + containersPreempted.value()
- + ", surplusContainers=" + surplusContainers.value() + '}';
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
deleted file mode 100644
index 0d11be2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public abstract class AbstractClientProvider {
-
- public AbstractClientProvider() {
- }
-
- /**
- * Generates a fixed format of application tags given one or more of
- * application name, version and description. This allows subsequent query for
- * an application with a name only, version only or description only or any
- * combination of those as filters.
- *
- * @param appName name of the application
- * @param appVersion version of the application
- * @param appDescription brief description of the application
- * @return
- */
- public static final Set<String> createApplicationTags(String appName,
- String appVersion, String appDescription) {
- Set<String> tags = new HashSet<>();
- tags.add(SliderUtils.createNameTag(appName));
- if (appVersion != null) {
- tags.add(SliderUtils.createVersionTag(appVersion));
- }
- if (appDescription != null) {
- tags.add(SliderUtils.createDescriptionTag(appDescription));
- }
- return tags;
- }
-
- /**
- * Validate the artifact.
- * @param artifact
- */
- public abstract void validateArtifact(Artifact artifact, FileSystem
- fileSystem) throws IOException;
-
- protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException;
-
- /**
- * Validate the config files.
- * @param configFiles config file list
- * @param fs file system
- */
- public void validateConfigFiles(List<ConfigFile> configFiles,
- FileSystem fs) throws IOException {
- Set<String> destFileSet = new HashSet<>();
-
- for (ConfigFile file : configFiles) {
- if (file.getType() == null) {
- throw new IllegalArgumentException("File type is empty");
- }
-
- if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
- .isEmpty(file.getSrcFile())) {
- throw new IllegalArgumentException(
- "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
-
- }
- if (!StringUtils.isEmpty(file.getSrcFile())) {
- Path p = new Path(file.getSrcFile());
- if (!fs.exists(p)) {
- throw new IllegalArgumentException(
- "Src_file does not exist for config file: " + file
- .getSrcFile());
- }
- }
-
- if (StringUtils.isEmpty(file.getDestFile())) {
- throw new IllegalArgumentException("Dest_file is empty.");
- }
-
- if (destFileSet.contains(file.getDestFile())) {
- throw new IllegalArgumentException(
- "Duplicated ConfigFile exists: " + file.getDestFile());
- }
- destFileSet.add(file.getDestFile());
-
- java.nio.file.Path destPath = Paths.get(file.getDestFile());
- if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
- throw new IllegalArgumentException("Non-absolute dest_file has more " +
- "than one path element");
- }
-
- // provider-specific validation
- validateConfigFile(file, fs);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
deleted file mode 100644
index 504680d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.provider;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
-import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_INTERVAL;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_MAX;
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
-
-public abstract class AbstractProviderService implements ProviderService,
- YarnServiceConstants {
-
- protected static final Logger log =
- LoggerFactory.getLogger(AbstractProviderService.class);
-
- public abstract void processArtifact(AbstractLauncher launcher,
- ComponentInstance compInstance, SliderFileSystem fileSystem,
- Application application)
- throws IOException;
-
- public void buildContainerLaunchContext(AbstractLauncher launcher,
- Application application, ComponentInstance instance,
- SliderFileSystem fileSystem, Configuration yarnConf)
- throws IOException, SliderException {
- Component component = instance.getComponent().getComponentSpec();;
- processArtifact(launcher, instance, fileSystem, application);
-
- ServiceContext context =
- instance.getComponent().getScheduler().getContext();
- // Generate tokens (key-value pair) for config substitution.
- // Get pre-defined tokens
- Map<String, String> globalTokens =
- instance.getComponent().getScheduler().globalTokens;
- Map<String, String> tokensForSubstitution = ProviderUtils
- .initCompTokensForSubstitute(instance);
- tokensForSubstitution.putAll(globalTokens);
- // Set the environment variables in launcher
- launcher.putEnv(SliderUtils
- .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
- launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
- launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- if (System.getenv(HADOOP_USER_NAME) != null) {
- launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
- }
- launcher.setEnv("LANG", "en_US.UTF-8");
- launcher.setEnv("LC_ALL", "en_US.UTF-8");
- launcher.setEnv("LANGUAGE", "en_US.UTF-8");
-
- for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
- tokensForSubstitution.put($(entry.getKey()), entry.getValue());
- }
- //TODO add component host tokens?
-// ProviderUtils.addComponentHostTokens(tokensForSubstitution, amState);
-
- // create config file on hdfs and add local resource
- ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
- component, tokensForSubstitution, instance, context);
-
- // substitute launch command
- String launchCommand = ProviderUtils
- .substituteStrWithTokens(component.getLaunchCommand(),
- tokensForSubstitution);
- CommandLineBuilder operation = new CommandLineBuilder();
- operation.add(launchCommand);
- operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
- launcher.addCommand(operation.build());
-
- // By default retry forever every 30 seconds
- launcher.setRetryContext(YarnServiceConf
- .getInt(CONTAINER_RETRY_MAX, -1, application.getConfiguration(),
- yarnConf), YarnServiceConf
- .getInt(CONTAINER_RETRY_INTERVAL, 30000, application.getConfiguration(),
- yarnConf));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/58] [abbrv] hadoop git commit: YARN-7202. Add UT for api-server.
Contributed by Eric Yang
Posted by ji...@apache.org.
YARN-7202. Add UT for api-server. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b57144a1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b57144a1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b57144a1
Branch: refs/heads/yarn-native-services
Commit: b57144a1726e0b02abbdc3e8f37769d3f7c039c8
Parents: b8a7ef1
Author: Jian He <ji...@apache.org>
Authored: Thu Oct 12 10:57:35 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:18 2017 -0800
----------------------------------------------------------------------
.../hadoop-yarn-services-api/pom.xml | 34 +-
.../hadoop/yarn/service/webapp/ApiServer.java | 21 +-
.../yarn/service/webapp/ApiServerWebApp.java | 2 +-
.../hadoop/yarn/service/ServiceClientTest.java | 107 ++++++
.../hadoop/yarn/service/TestApiServer.java | 366 +++++++++++++++++++
.../hadoop/yarn/service/ServiceTestUtils.java | 171 +++++++++
.../yarn/service/TestYarnNativeServices.java | 162 +-------
7 files changed, 690 insertions(+), 173 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 74d9681..ddea2a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -41,7 +41,6 @@
</resources>
<plugins>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
@@ -66,17 +65,6 @@
</execution>
</executions>
</plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <environmentVariables>
- <JAVA_HOME>${java.home}</JAVA_HOME>
- </environmentVariables>
- </configuration>
- </plugin>
-
</plugins>
</build>
@@ -84,6 +72,7 @@
</reporting>
<dependencies>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services-core</artifactId>
@@ -116,5 +105,26 @@
<groupId>javax.ws.rs</groupId>
<artifactId>jsr311-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- Test dependencies -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 4fe12e3..5773069 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -165,12 +165,12 @@ public class ApiServer {
} else {
LOG.info("Successfully stopped service {}", appName);
}
- return Response.status(Status.NO_CONTENT).build();
+ return Response.status(Status.OK).build();
} catch (ApplicationNotFoundException e) {
ServiceStatus serviceStatus = new ServiceStatus();
serviceStatus.setDiagnostics(
- "Service " + appName + " not found " + e.getMessage());
- return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+ "Service " + appName + " is not found in YARN: " + e.getMessage());
+ return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
.build();
} catch (Exception e) {
ServiceStatus serviceStatus = new ServiceStatus();
@@ -245,7 +245,8 @@ public class ApiServer {
// flex a single component app
if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil
.hasComponent(updateServiceData)) {
- Component defaultComp = ServiceApiUtil.createDefaultComponent(updateServiceData);
+ Component defaultComp = ServiceApiUtil
+ .createDefaultComponent(updateServiceData);
return updateComponent(updateServiceData.getName(), defaultComp.getName(),
defaultComp);
}
@@ -291,4 +292,16 @@ public class ApiServer {
.entity(status).build();
}
}
+
+ /**
+ * Used by negative test case.
+ *
+ * @param mockServerClient - A mocked version of ServiceClient
+ */
+ public static void setServiceClient(ServiceClient mockServerClient) {
+ SERVICE_CLIENT = mockServerClient;
+ SERVICE_CLIENT.init(YARN_CONFIG);
+ SERVICE_CLIENT.start();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
index fc65a63..f4acd94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
@@ -74,7 +74,7 @@ public class ApiServerWebApp extends AbstractService {
@Override
protected void serviceStart() throws Exception {
bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS,
- DEFAULT_API_SERVER_ADDRESS , DEFAULT_API_SERVER_PORT);
+ DEFAULT_API_SERVER_ADDRESS, DEFAULT_API_SERVER_PORT);
logger.info("YARN API server running on " + bindAddress);
if (UserGroupInformation.isSecurityEnabled()) {
doSecureLogin(getConfig());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
new file mode 100644
index 0000000..3e08c3a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+
+/**
+ * A mock version of ServiceClient - This class is design
+ * to simulate various error conditions that will happen
+ * when a consumer class calls ServiceClient.
+ */
+public class ServiceClientTest extends ServiceClient {
+
+ private Configuration conf = new Configuration();
+
+ protected static void init() {
+ }
+
+ public ServiceClientTest() {
+ super();
+ }
+
+ @Override
+ public Configuration getConfig() {
+ return conf;
+ }
+
+ @Override
+ public ApplicationId actionCreate(Service service) {
+ String serviceName = service.getName();
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ return ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ }
+
+ @Override
+ public Service getStatus(String appName) {
+ if (appName == null) {
+ throw new NullPointerException();
+ }
+ if (appName.equals("jenkins")) {
+ return new Service();
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ @Override
+ public int actionStart(String serviceName)
+ throws YarnException, IOException {
+ if (serviceName == null) {
+ throw new NullPointerException();
+ }
+ if (serviceName.equals("jenkins")) {
+ return EXIT_SUCCESS;
+ } else {
+ throw new ApplicationNotFoundException("");
+ }
+ }
+
+ @Override
+ public int actionStop(String serviceName, boolean waitForAppStopped)
+ throws YarnException, IOException {
+ if (serviceName == null) {
+ throw new NullPointerException();
+ }
+ if (serviceName.equals("jenkins")) {
+ return EXIT_SUCCESS;
+ } else {
+ throw new ApplicationNotFoundException("");
+ }
+ }
+
+ @Override
+ public int actionDestroy(String serviceName) {
+ if (serviceName == null) {
+ throw new NullPointerException();
+ }
+ if (serviceName.equals("jenkins")) {
+ return EXIT_SUCCESS;
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
new file mode 100644
index 0000000..2b22474
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.webapp.ApiServer;
+import javax.ws.rs.Path;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test case for ApiServer REST API.
+ *
+ */
+public class TestApiServer {
+ private ApiServer apiServer;
+
+ @Before
+ public void setup() throws Exception {
+ ServiceClient mockServerClient = new ServiceClientTest();
+ Configuration conf = new Configuration();
+ conf.set("yarn.api-service.service.client.class",
+ ServiceClientTest.class.getName());
+ ApiServer.setServiceClient(mockServerClient);
+ this.apiServer = new ApiServer(conf);
+ }
+
+ @Test
+ public void testPathAnnotation() {
+ assertNotNull(this.apiServer.getClass().getAnnotation(Path.class));
+ assertTrue("The controller has the annotation Path",
+ this.apiServer.getClass().isAnnotationPresent(Path.class));
+ final Path path = this.apiServer.getClass()
+ .getAnnotation(Path.class);
+ assertEquals("The path has /ws/v1 annotation", path.value(),
+ "/ws/v1");
+ }
+
+ @Test
+ public void testGetVersion() {
+ final Response actual = apiServer.getVersion();
+ assertEquals("Version number is", actual.getStatus(),
+ Response.ok().build().getStatus());
+ }
+
+ @Test
+ public void testBadCreateService() {
+ Service service = new Service();
+ // Test for invalid argument
+ final Response actual = apiServer.createService(service);
+ assertEquals("Create service is ", actual.getStatus(),
+ Response.status(Status.BAD_REQUEST).build().getStatus());
+ }
+
+ @Test
+ public void testGoodCreateService() {
+ Service service = new Service();
+ service.setName("jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(1L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.createService(service);
+ assertEquals("Create service is ", actual.getStatus(),
+ Response.status(Status.ACCEPTED).build().getStatus());
+ }
+
+ @Test
+ public void testBadGetService() {
+ final Response actual = apiServer.getService("no-jenkins");
+ assertEquals("Get service is ", actual.getStatus(),
+ Response.status(Status.NOT_FOUND).build().getStatus());
+ }
+
+ @Test
+ public void testBadGetService2() {
+ final Response actual = apiServer.getService(null);
+ assertEquals("Get service is ", actual.getStatus(),
+ Response.status(Status.INTERNAL_SERVER_ERROR)
+ .build().getStatus());
+ }
+
+ @Test
+ public void testGoodGetService() {
+ final Response actual = apiServer.getService("jenkins");
+ assertEquals("Get service is ", actual.getStatus(),
+ Response.status(Status.OK).build().getStatus());
+ }
+
+ @Test
+ public void testBadDeleteService() {
+ final Response actual = apiServer.deleteService("no-jenkins");
+ assertEquals("Delete service is ", actual.getStatus(),
+ Response.status(Status.BAD_REQUEST).build().getStatus());
+ }
+
+ @Test
+ public void testBadDeleteService2() {
+ final Response actual = apiServer.deleteService(null);
+ assertEquals("Delete service is ", actual.getStatus(),
+ Response.status(Status.INTERNAL_SERVER_ERROR)
+ .build().getStatus());
+ }
+
+ @Test
+ public void testGoodDeleteService() {
+ final Response actual = apiServer.deleteService("jenkins");
+ assertEquals("Delete service is ", actual.getStatus(),
+ Response.status(Status.OK).build().getStatus());
+ }
+
+ @Test
+ public void testDecreaseContainerAndStop() {
+ Service service = new Service();
+ service.setState(ServiceState.STOPPED);
+ service.setName("jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(0L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.updateService("jenkins",
+ service);
+ assertEquals("update service is ", actual.getStatus(),
+ Response.status(Status.OK).build().getStatus());
+ }
+
+ @Test
+ public void testBadDecreaseContainerAndStop() {
+ Service service = new Service();
+ service.setState(ServiceState.STOPPED);
+ service.setName("no-jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("no-jenkins");
+ c.setNumberOfContainers(-1L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ System.out.println("before stop");
+ final Response actual = apiServer.updateService("no-jenkins",
+ service);
+ assertEquals("flex service is ", actual.getStatus(),
+ Response.status(Status.BAD_REQUEST).build().getStatus());
+ }
+
+ @Test
+ public void testIncreaseContainersAndStart() {
+ Service service = new Service();
+ service.setState(ServiceState.STARTED);
+ service.setName("jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(2L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.updateService("jenkins",
+ service);
+ assertEquals("flex service is ", actual.getStatus(),
+ Response.status(Status.OK).build().getStatus());
+ }
+
+ @Test
+ public void testBadStartServices() {
+ Service service = new Service();
+ service.setState(ServiceState.STARTED);
+ service.setName("no-jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(2L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.updateService("no-jenkins",
+ service);
+ assertEquals("start service is ", actual.getStatus(),
+ Response.status(Status.INTERNAL_SERVER_ERROR).build()
+ .getStatus());
+ }
+
+ @Test
+ public void testGoodStartServices() {
+ Service service = new Service();
+ service.setState(ServiceState.STARTED);
+ service.setName("jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(2L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.updateService("jenkins",
+ service);
+ assertEquals("start service is ", actual.getStatus(),
+ Response.status(Status.OK).build().getStatus());
+ }
+
+ @Test
+ public void testBadStopServices() {
+ Service service = new Service();
+ service.setState(ServiceState.STOPPED);
+ service.setName("no-jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("no-jenkins");
+ c.setNumberOfContainers(-1L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ System.out.println("before stop");
+ final Response actual = apiServer.updateService("no-jenkins",
+ service);
+ assertEquals("stop service is ", actual.getStatus(),
+ Response.status(Status.BAD_REQUEST).build().getStatus());
+ }
+
+ @Test
+ public void testGoodStopServices() {
+ Service service = new Service();
+ service.setState(ServiceState.STARTED);
+ service.setName("jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(-1L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ System.out.println("before stop");
+ final Response actual = apiServer.updateService("jenkins",
+ service);
+ assertEquals("stop service is ", actual.getStatus(),
+ Response.status(Status.OK).build().getStatus());
+ }
+
+ @Test
+ public void testUpdateService() {
+ Service service = new Service();
+ service.setState(ServiceState.STARTED);
+ service.setName("no-jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("no-jenkins");
+ c.setNumberOfContainers(-1L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ System.out.println("before stop");
+ final Response actual = apiServer.updateService("no-jenkins",
+ service);
+ assertEquals("update service is ", actual.getStatus(),
+ Response.status(Status.INTERNAL_SERVER_ERROR)
+ .build().getStatus());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index a2edbc8..aeb4341 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -18,24 +18,58 @@
package org.apache.hadoop.yarn.service;
+import org.apache.commons.io.FileUtils;
+import org.apache.curator.test.TestingCluster;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URL;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_PMEM_CHECK_ENABLED;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_VMEM_CHECK_ENABLED;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_SERVICE_ENABLED;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class ServiceTestUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceTestUtils.class);
+
+ private MiniYARNCluster yarnCluster = null;
+ private MiniDFSCluster hdfsCluster = null;
+ private FileSystem fs = null;
+ private Configuration conf = null;
+ public static final int NUM_NMS = 1;
+ private File basedir;
+
public static final JsonSerDeser<Service> JSON_SER_DESER =
new JsonSerDeser<>(Service.class,
PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
@@ -84,4 +118,141 @@ public class ServiceTestUtils {
ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
return sfs;
}
+
+ protected void setConf(YarnConfiguration conf) {
+ this.conf = conf;
+ }
+
+ protected Configuration getConf() {
+ return conf;
+ }
+
+ protected FileSystem getFS() {
+ return fs;
+ }
+
+ protected void setupInternal(int numNodeManager)
+ throws Exception {
+ LOG.info("Starting up YARN cluster");
+// Logger rootLogger = LogManager.getRootLogger();
+// rootLogger.setLevel(Level.DEBUG);
+ setConf(new YarnConfiguration());
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+ // reduce the teardown waiting time
+ conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
+ conf.set("yarn.log.dir", "target");
+ // mark if we need to launch the v1 timeline server
+ // disable aux-service based timeline aggregators
+ conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
+ conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
+ // Enable ContainersMonitorImpl
+ conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
+ LinuxResourceCalculatorPlugin.class.getName());
+ conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
+ ProcfsBasedProcessTree.class.getName());
+ conf.setBoolean(
+ YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true);
+ conf.setBoolean(TIMELINE_SERVICE_ENABLED, false);
+ conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
+ conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000);
+ conf.setLong(AM_RESOURCE_MEM, 526);
+ conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5);
+ // Disable vmem check to disallow NM killing the container
+ conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
+ conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
+ // setup zk cluster
+ TestingCluster zkCluster;
+ zkCluster = new TestingCluster(1);
+ zkCluster.start();
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
+ conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
+ LOG.info("ZK cluster: " + zkCluster.getConnectString());
+
+ fs = FileSystem.get(conf);
+ basedir = new File("target", "apps");
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+
+ if (yarnCluster == null) {
+ yarnCluster =
+ new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1,
+ numNodeManager, 1, 1);
+ yarnCluster.init(conf);
+ yarnCluster.start();
+
+ waitForNMsToRegister();
+
+ URL url = Thread.currentThread().getContextClassLoader()
+ .getResource("yarn-site.xml");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find 'yarn-site.xml' dummy file in classpath");
+ }
+ Configuration yarnClusterConfig = yarnCluster.getConfig();
+ yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ new File(url.getPath()).getParent());
+ //write the document to a buffer (not directly to the file, as that
+ //can cause the file being written to get read -which will then fail.
+ ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+ yarnClusterConfig.writeXml(bytesOut);
+ bytesOut.close();
+ //write the bytes to the file in the classpath
+ OutputStream os = new FileOutputStream(new File(url.getPath()));
+ os.write(bytesOut.toByteArray());
+ os.close();
+ LOG.info("Write yarn-site.xml configs to: " + url);
+ }
+ if (hdfsCluster == null) {
+ HdfsConfiguration hdfsConfig = new HdfsConfiguration();
+ hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
+ .numDataNodes(1).build();
+ }
+
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
+ }
+ }
+
+ public void shutdown() throws IOException {
+ if (yarnCluster != null) {
+ try {
+ yarnCluster.stop();
+ } finally {
+ yarnCluster = null;
+ }
+ }
+ if (hdfsCluster != null) {
+ try {
+ hdfsCluster.shutdown();
+ } finally {
+ hdfsCluster = null;
+ }
+ }
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ SliderFileSystem sfs = new SliderFileSystem(conf);
+ Path appDir = sfs.getBaseApplicationPath();
+ sfs.getFileSystem().delete(appDir, true);
+ }
+
+ private void waitForNMsToRegister() throws Exception {
+ int sec = 60;
+ while (sec >= 0) {
+ if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size()
+ >= NUM_NMS) {
+ break;
+ }
+ Thread.sleep(1000);
+ sec--;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 30f2aeb..d821b84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -18,31 +18,20 @@
package org.apache.hadoop.yarn.service;
-import org.apache.commons.io.FileUtils;
-import org.apache.curator.test.TestingCluster;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
-import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -52,12 +41,7 @@ import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -66,28 +50,18 @@ import java.util.Map;
import java.util.TreeSet;
import java.util.concurrent.TimeoutException;
-import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
* End to end tests to test deploying services with MiniYarnCluster and a in-JVM
* ZK testing cluster.
*/
-public class TestYarnNativeServices extends ServiceTestUtils{
+public class TestYarnNativeServices extends ServiceTestUtils {
private static final Logger LOG =
LoggerFactory.getLogger(TestYarnNativeServices.class);
- private MiniYARNCluster yarnCluster = null;
- private MiniDFSCluster hdfsCluster = null;
- private FileSystem fs = null;
- protected Configuration conf = null;
- private static final int NUM_NMS = 1;
- private File basedir;
-
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
@@ -96,135 +70,11 @@ public class TestYarnNativeServices extends ServiceTestUtils{
setupInternal(NUM_NMS);
}
- private void setupInternal(int numNodeManager)
- throws Exception {
- LOG.info("Starting up YARN cluster");
-// Logger rootLogger = LogManager.getRootLogger();
-// rootLogger.setLevel(Level.DEBUG);
- conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
- // reduce the teardown waiting time
- conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
- conf.set("yarn.log.dir", "target");
- // mark if we need to launch the v1 timeline server
- // disable aux-service based timeline aggregators
- conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
- conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
- // Enable ContainersMonitorImpl
- conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
- LinuxResourceCalculatorPlugin.class.getName());
- conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
- ProcfsBasedProcessTree.class.getName());
- conf.setBoolean(
- YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true);
- conf.setBoolean(TIMELINE_SERVICE_ENABLED, false);
- conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
- conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000);
- conf.setLong(AM_RESOURCE_MEM, 526);
- conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5);
- // Disable vmem check to disallow NM killing the container
- conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
- conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
- // setup zk cluster
- TestingCluster zkCluster;
- zkCluster = new TestingCluster(1);
- zkCluster.start();
- conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
- conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
- LOG.info("ZK cluster: " + zkCluster.getConnectString());
-
- fs = FileSystem.get(conf);
- basedir = new File("target", "apps");
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
-
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
-
- if (yarnCluster == null) {
- yarnCluster =
- new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1,
- numNodeManager, 1, 1);
- yarnCluster.init(conf);
- yarnCluster.start();
-
- waitForNMsToRegister();
-
- URL url = Thread.currentThread().getContextClassLoader()
- .getResource("yarn-site.xml");
- if (url == null) {
- throw new RuntimeException(
- "Could not find 'yarn-site.xml' dummy file in classpath");
- }
- Configuration yarnClusterConfig = yarnCluster.getConfig();
- yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- new File(url.getPath()).getParent());
- //write the document to a buffer (not directly to the file, as that
- //can cause the file being written to get read -which will then fail.
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- yarnClusterConfig.writeXml(bytesOut);
- bytesOut.close();
- //write the bytes to the file in the classpath
- OutputStream os = new FileOutputStream(new File(url.getPath()));
- os.write(bytesOut.toByteArray());
- os.close();
- LOG.info("Write yarn-site.xml configs to: " + url);
- }
- if (hdfsCluster == null) {
- HdfsConfiguration hdfsConfig = new HdfsConfiguration();
- hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
- .numDataNodes(1).build();
- }
-
- try {
- Thread.sleep(2000);
- } catch (InterruptedException e) {
- LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
- }
-
-
- }
-
- private void waitForNMsToRegister() throws Exception {
- int sec = 60;
- while (sec >= 0) {
- if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size()
- >= NUM_NMS) {
- break;
- }
- Thread.sleep(1000);
- sec--;
- }
- }
-
@After
public void tearDown() throws IOException {
- if (yarnCluster != null) {
- try {
- yarnCluster.stop();
- } finally {
- yarnCluster = null;
- }
- }
- if (hdfsCluster != null) {
- try {
- hdfsCluster.shutdown();
- } finally {
- hdfsCluster = null;
- }
- }
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- SliderFileSystem sfs = new SliderFileSystem(conf);
- Path appDir = sfs.getBaseApplicationPath();
- sfs.getFileSystem().delete(appDir, true);
+ shutdown();
}
-
-
// End-to-end test to use ServiceClient to deploy a service.
// 1. Create a service with 2 components, each of which has 2 containers
// 2. Flex up each component to 3 containers and check the component instance names
@@ -237,11 +87,11 @@ public class TestYarnNativeServices extends ServiceTestUtils{
ServiceClient client = createClient();
Service exampleApp = createExampleApplication();
client.actionCreate(exampleApp);
- SliderFileSystem fileSystem = new SliderFileSystem(conf);
+ SliderFileSystem fileSystem = new SliderFileSystem(getConf());
Path appDir = fileSystem.buildClusterDirPath(exampleApp.getName());
// check app.json is persisted.
Assert.assertTrue(
- fs.exists(new Path(appDir, exampleApp.getName() + ".json")));
+ getFS().exists(new Path(appDir, exampleApp.getName() + ".json")));
waitForAllCompToBeReady(client, exampleApp);
// Flex two components, each from 2 container to 3 containers.
@@ -277,7 +127,7 @@ public class TestYarnNativeServices extends ServiceTestUtils{
//destroy the service and check the app dir is deleted from fs.
client.actionDestroy(exampleApp.getName());
// check the service dir on hdfs (in this case, local fs) are deleted.
- Assert.assertFalse(fs.exists(appDir));
+ Assert.assertFalse(getFS().exists(appDir));
}
// Create compa with 2 containers
@@ -456,7 +306,7 @@ public class TestYarnNativeServices extends ServiceTestUtils{
return null;
}
};
- client.init(conf);
+ client.init(getConf());
client.start();
return client;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/58] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd
and merge service cmd into application cmd. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index f22d487..df4b1df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -20,88 +20,71 @@ package org.apache.hadoop.yarn.service.client;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Mockito.*;
public class TestServiceCLI {
+ private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI
+ .class);
- protected Configuration conf = new YarnConfiguration();
+ private Configuration conf = new YarnConfiguration();
private File basedir;
- private ServiceCLI cli;
private SliderFileSystem fs;
+ private String basedirProp;
- private void buildApp(String appDef) throws Throwable {
- String[] args =
- { "build", ARG_FILE, ExampleAppJson.resourceName(appDef)};
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- cli.exec(clientArgs);
+ private void runCLI(String[] args) throws Exception {
+ LOG.info("running CLI: yarn {}", Arrays.asList(args));
+ ApplicationCLI cli = new ApplicationCLI();
+ cli.setSysOutPrintStream(System.out);
+ cli.setSysErrPrintStream(System.err);
+ int res = ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+ cli.stop();
+ }
+
+ private void buildApp(String serviceName, String appDef) throws Throwable {
+ String[] args = {"app", "-D", basedirProp, "-save", serviceName,
+ ExampleAppJson.resourceName(appDef)};
+ runCLI(args);
+ }
+
+ private void buildApp(String serviceName, String appDef, String lifetime,
+ String queue) throws Throwable {
+ String[] args = {"app", "-D", basedirProp, "-save", serviceName,
+ ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime,
+ "-changeQueue", queue};
+ runCLI(args);
}
@Before
public void setup() throws Throwable {
basedir = new File("target", "apps");
+ basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath();
conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
- conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0);
- conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1);
- conf.setInt(
- CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- conf.setInt(CommonConfigurationKeysPublic.
- IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
fs = new SliderFileSystem(conf);
if (basedir.exists()) {
FileUtils.deleteDirectory(basedir);
} else {
basedir.mkdirs();
}
-
- // create a CLI and skip connection to AM
- cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient() {
- @Override
- protected void serviceInit(Configuration configuration)
- throws Exception {
- super.serviceInit(conf);
- yarnClient = spy(yarnClient);
- ApplicationReport report = Records.newRecord(ApplicationReport.class);
- report.setYarnApplicationState(YarnApplicationState.RUNNING);
- report.setHost("localhost");
- doReturn(report).when(yarnClient).getApplicationReport(anyObject());
- }
- @Override
- protected ClientAMProtocol createAMProxy(String host, int port)
- throws IOException {
- return mock(ClientAMProtocol.class);
- }
- };
- client.init(conf);
- client.start();
- }
- };
}
@After
@@ -111,41 +94,31 @@ public class TestServiceCLI {
}
}
- // Test flex components count are persisted.
@Test
public void testFlexComponents() throws Throwable {
+ // currently can only test building apps, since that is the only
+ // operation that doesn't require an RM
+ // TODO: expand CLI test to try other commands
String serviceName = "app-1";
- buildApp(ExampleAppJson.APP_JSON);
- checkCompCount("master",serviceName, 1L);
-
- // increase by 2
- String[] flexUpArgs = {"flex", serviceName, "--component", "master" , "+2"};
- ClientArgs clientArgs = new ClientArgs(flexUpArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
- checkCompCount("master", serviceName, 3L);
-
- // decrease by 1
- String[] flexDownArgs = {"flex", serviceName, "--component", "master", "-1"};
- clientArgs = new ClientArgs(flexDownArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
- checkCompCount("master", serviceName, 2L);
+ buildApp(serviceName, ExampleAppJson.APP_JSON);
+ checkApp(serviceName, "master", 1L, 3600L, null);
- String[] flexAbsoluteArgs = {"flex", serviceName, "--component", "master", "10"};
- clientArgs = new ClientArgs(flexAbsoluteArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
- checkCompCount("master", serviceName, 10L);
+ serviceName = "app-2";
+ buildApp(serviceName, ExampleAppJson.APP_JSON, "1000", "qname");
+ checkApp(serviceName, "master", 1L, 1000L, "qname");
}
- private void checkCompCount(String compName, String serviceName, long count)
- throws IOException {
- List<Component> components =
- ServiceApiUtil.getComponents(fs, serviceName);
+ private void checkApp(String serviceName, String compName, long count, Long
+ lifetime, String queue) throws IOException {
+ Service service = ServiceApiUtil.loadService(fs, serviceName);
+ Assert.assertEquals(serviceName, service.getName());
+ Assert.assertEquals(lifetime, service.getLifetime());
+ Assert.assertEquals(queue, service.getQueue());
+ List<Component> components = service.getComponents();
for (Component component : components) {
if (component.getName().equals(compName)) {
- Assert.assertEquals(count, component.getNumberOfContainers().longValue());
+ Assert.assertEquals(count, component.getNumberOfContainers()
+ .longValue());
return;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
new file mode 100644
index 0000000..6310178
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.api;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Client for managing applications.
+ */
+@Public
+@Unstable
+public abstract class AppAdminClient extends CompositeService {
+ public static final String YARN_APP_ADMIN_CLIENT_PREFIX = "yarn" +
+ ".application.admin.client.class.";
+ public static final String DEFAULT_TYPE = "yarn-service";
+ public static final String DEFAULT_CLASS_NAME = "org.apache.hadoop.yarn" +
+ ".service.client.ServiceClient";
+
+ @Private
+ protected AppAdminClient() {
+ super(AppAdminClient.class.getName());
+ }
+
+ /**
+ * <p>
+ * Create a new instance of AppAdminClient.
+ * </p>
+ *
+ * @param appType application type
+ * @param conf configuration
+ * @return app admin client
+ */
+ @Public
+ @Unstable
+ public static AppAdminClient createAppAdminClient(String appType,
+ Configuration conf) {
+ Map<String, String> clientClassMap =
+ conf.getPropsWithPrefix(YARN_APP_ADMIN_CLIENT_PREFIX);
+ if (!clientClassMap.containsKey(DEFAULT_TYPE)) {
+ clientClassMap.put(DEFAULT_TYPE, DEFAULT_CLASS_NAME);
+ }
+ if (!clientClassMap.containsKey(appType)) {
+ throw new IllegalArgumentException("App admin client class name not " +
+ "specified for type " + appType);
+ }
+ String clientClassName = clientClassMap.get(appType);
+ Class<? extends AppAdminClient> clientClass;
+ try {
+ clientClass = (Class<? extends AppAdminClient>) Class.forName(
+ clientClassName);
+ } catch (ClassNotFoundException e) {
+ throw new YarnRuntimeException("Invalid app admin client class", e);
+ }
+
+ AppAdminClient appAdminClient = ReflectionUtils.newInstance(clientClass,
+ conf);
+ appAdminClient.init(conf);
+ appAdminClient.start();
+ return appAdminClient;
+ }
+
+ /**
+ * <p>
+ * Launch a new YARN application.
+ * </p>
+ *
+ * @param fileName specification of application
+ * @param appName name of the application
+ * @param lifetime lifetime of the application
+ * @param queue queue of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionLaunch(String fileName, String appName, Long
+ lifetime, String queue) throws IOException, YarnException;
+
+ /**
+ * <p>
+ * Stop a YARN application (attempt to stop gracefully before killing the
+ * application). In the case of a long-running service, the service may be
+ * restarted later.
+ * </p>
+ *
+ * @param appName the name of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionStop(String appName) throws IOException,
+ YarnException;
+
+ /**
+ * <p>
+ * Start a YARN application from a previously saved specification. In the
+ * case of a long-running service, the service must have been previously
+ * launched/started and then stopped, or previously saved but not started.
+ * </p>
+ *
+ * @param appName the name of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionStart(String appName) throws IOException,
+ YarnException;
+
+ /**
+ * <p>
+ * Save the specification for a YARN application / long-running service.
+ * The application may be started later.
+ * </p>
+ *
+ * @param fileName specification of application to save
+ * @param appName name of the application
+ * @param lifetime lifetime of the application
+ * @param queue queue of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionSave(String fileName, String appName, Long
+ lifetime, String queue) throws IOException, YarnException;
+
+ /**
+ * <p>
+ * Remove the specification and all application data for a YARN application.
+ * The application cannot be running.
+ * </p>
+ *
+ * @param appName the name of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionDestroy(String appName) throws IOException,
+ YarnException;
+
+ /**
+ * <p>
+ * Change the number of running containers for a component of a YARN
+ * application / long-running service.
+ * </p>
+ *
+ * @param appName the name of the application
+ * @param componentCounts map of component name to new component count or
+ * amount to change existing component count (e.g.
+ * 5, +5, -5)
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionFlex(String appName, Map<String, String>
+ componentCounts) throws IOException, YarnException;
+
+ /**
+ * <p>
+ * Upload AM dependencies to HDFS. This makes future application launches
+ * faster since the dependencies do not have to be uploaded on each launch.
+ * </p>
+ *
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int enableFastLaunch() throws IOException,
+ YarnException;
+
+ /**
+ * <p>
+ * Get detailed status string for a YARN application.
+ * </p>
+ *
+ * @param applicationId application id
+ * @return status string
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract String getStatusString(String applicationId) throws
+ IOException, YarnException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 2a9b3bc..fb08fcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -23,12 +23,7 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.text.DecimalFormat;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
@@ -54,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -85,6 +81,7 @@ public class ApplicationCLI extends YarnCLI {
"%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ System.getProperty("line.separator");
+ public static final String APP = "app";
public static final String APPLICATION = "application";
public static final String APPLICATION_ATTEMPT = "applicationattempt";
public static final String CONTAINER = "container";
@@ -93,22 +90,52 @@ public class ApplicationCLI extends YarnCLI {
public static final String UPDATE_LIFETIME = "updateLifetime";
public static final String CHANGE_APPLICATION_QUEUE = "changeQueue";
+ // app admin options
+ public static final String LAUNCH_CMD = "launch";
+ public static final String STOP_CMD = "stop";
+ public static final String START_CMD = "start";
+ public static final String SAVE_CMD = "save";
+ public static final String DESTROY_CMD = "destroy";
+ public static final String FLEX_CMD = "flex";
+ public static final String COMPONENT = "component";
+ public static final String ENABLE_FAST_LAUNCH = "enableFastLaunch";
+
+ private static String firstArg = null;
+
private boolean allAppStates;
public static void main(String[] args) throws Exception {
ApplicationCLI cli = new ApplicationCLI();
cli.setSysOutPrintStream(System.out);
cli.setSysErrPrintStream(System.err);
- int res = ToolRunner.run(cli, args);
+ int res = ToolRunner.run(cli, preProcessArgs(args));
cli.stop();
System.exit(res);
}
+ @VisibleForTesting
+ public static String[] preProcessArgs(String[] args) {
+ if (args.length > 0) {
+ // first argument (app|application|applicationattempt|container) must
+ // be stripped off for GenericOptionsParser to work
+ firstArg = args[0];
+ return Arrays.copyOfRange(args, 1, args.length);
+ } else {
+ return args;
+ }
+ }
+
@Override
public int run(String[] args) throws Exception {
Options opts = new Options();
String title = null;
- if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION)) {
+ if (firstArg != null) {
+ title = firstArg;
+ } else if (args.length > 0) {
+ title = args[0];
+ }
+ if (title != null && (title.equalsIgnoreCase(APPLICATION) || title
+ .equalsIgnoreCase(APP))) {
title = APPLICATION;
opts.addOption(STATUS_CMD, true,
"Prints the status of the application.");
@@ -168,8 +195,52 @@ public class ApplicationCLI extends YarnCLI {
opts.getOption(UPDATE_PRIORITY).setArgName("Priority");
opts.getOption(UPDATE_LIFETIME).setArgName("Timeout");
opts.getOption(CHANGE_APPLICATION_QUEUE).setArgName("Queue Name");
- } else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
- title = APPLICATION_ATTEMPT;
+ opts.addOption(LAUNCH_CMD, true, "Launches application from " +
+ "specification file (saves specification and starts application). " +
+ "Options -updateLifetime and -changeQueue can be specified to alter" +
+ " the values provided in the file. Supports -appTypes option to " +
+ "specify which client implementation to use.");
+ opts.addOption(STOP_CMD, true, "Stops application gracefully (may be " +
+ "started again later). If name is provided, appType must be " +
+ "provided unless it is the default yarn-service. If ID is provided," +
+ " the appType will be looked up. Supports -appTypes option to " +
+ "specify which client implementation to use.");
+ opts.addOption(START_CMD, true, "Starts a previously saved " +
+ "application. Supports -appTypes option to specify which client " +
+ "implementation to use.");
+ opts.addOption(SAVE_CMD, true, "Saves specification file for " +
+ "an application. Options -updateLifetime and -changeQueue can be " +
+ "specified to alter the values provided in the file. Supports " +
+ "-appTypes option to specify which client implementation to use.");
+ opts.addOption(DESTROY_CMD, true, "Destroys a saved application " +
+ "specification and removes all application data permanently. " +
+ "Supports -appTypes option to specify which client implementation " +
+ "to use.");
+ opts.addOption(FLEX_CMD, true, "Changes number of " +
+ "running containers for a component of an application / " +
+ "long-running service. Requires -component option. If name is " +
+ "provided, appType must be provided unless it is the default " +
+ "yarn-service. If ID is provided, the appType will be looked up. " +
+ "Supports -appTypes option to specify which client implementation " +
+ "to use.");
+ opts.addOption(COMPONENT, true, "Works with -flex option to change " +
+ "the number of components/containers running for an application / " +
+ "long-running service. Supports absolute or relative changes, such " +
+ "as +1, 2, or -3.");
+ opts.addOption(ENABLE_FAST_LAUNCH, false, "Uploads AM dependencies " +
+ "to HDFS to make future launches faster. Supports -appTypes option" +
+ " to specify which client implementation to use.");
+ opts.getOption(LAUNCH_CMD).setArgName("Application Name> <File Name");
+ opts.getOption(LAUNCH_CMD).setArgs(2);
+ opts.getOption(START_CMD).setArgName("Application Name");
+ opts.getOption(STOP_CMD).setArgName("Application Name or ID");
+ opts.getOption(SAVE_CMD).setArgName("Application Name> <File Name");
+ opts.getOption(SAVE_CMD).setArgs(2);
+ opts.getOption(DESTROY_CMD).setArgName("Application Name");
+ opts.getOption(FLEX_CMD).setArgName("Application Name or ID");
+ opts.getOption(COMPONENT).setArgName("Component Name> <Count");
+ opts.getOption(COMPONENT).setArgs(2);
+ } else if (title != null && title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
opts.addOption(STATUS_CMD, true,
"Prints the status of the application attempt.");
opts.addOption(LIST_CMD, true,
@@ -179,8 +250,7 @@ public class ApplicationCLI extends YarnCLI {
opts.getOption(STATUS_CMD).setArgName("Application Attempt ID");
opts.getOption(LIST_CMD).setArgName("Application ID");
opts.getOption(FAIL_CMD).setArgName("Application Attempt ID");
- } else if (args.length > 0 && args[0].equalsIgnoreCase(CONTAINER)) {
- title = CONTAINER;
+ } else if (title != null && title.equalsIgnoreCase(CONTAINER)) {
opts.addOption(STATUS_CMD, true,
"Prints the status of the container.");
opts.addOption(LIST_CMD, true,
@@ -205,23 +275,53 @@ public class ApplicationCLI extends YarnCLI {
printUsage(title, opts);
return exitCode;
}
+ String[] unparsedArgs = cliParser.getArgs();
+ if (firstArg == null) {
+ if (unparsedArgs.length != 1) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ } else {
+ if (unparsedArgs.length != 0) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ }
if (cliParser.hasOption(STATUS_CMD)) {
- if (args.length != 3) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, STATUS_CMD)) {
printUsage(title, opts);
return exitCode;
}
- if (args[0].equalsIgnoreCase(APPLICATION)) {
- exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
- } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
+ if (title.equalsIgnoreCase(APPLICATION) ||
+ title.equalsIgnoreCase(APP)) {
+ ApplicationReport report = printApplicationReport(cliParser
+ .getOptionValue(STATUS_CMD));
+ if (report == null) {
+ exitCode = -1;
+ } else {
+ exitCode = 0;
+ String appType = report.getApplicationType();
+ try {
+ AppAdminClient client = AppAdminClient.createAppAdminClient(appType,
+ getConf());
+ sysout.println("Detailed Application Status :");
+ sysout.println(client.getStatusString(cliParser.getOptionValue(
+ STATUS_CMD)));
+ } catch (IllegalArgumentException e) {
+ // app type does not have app admin client implementation
+ }
+ }
+ } else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
exitCode = printApplicationAttemptReport(cliParser
.getOptionValue(STATUS_CMD));
- } else if (args[0].equalsIgnoreCase(CONTAINER)) {
+ } else if (title.equalsIgnoreCase(CONTAINER)) {
exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD));
}
return exitCode;
} else if (cliParser.hasOption(LIST_CMD)) {
- if (args[0].equalsIgnoreCase(APPLICATION)) {
+ if (title.equalsIgnoreCase(APPLICATION) ||
+ title.equalsIgnoreCase(APP)) {
allAppStates = false;
Set<String> appTypes = new HashSet<String>();
if (cliParser.hasOption(APP_TYPE_CMD)) {
@@ -272,21 +372,21 @@ public class ApplicationCLI extends YarnCLI {
}
}
listApplications(appTypes, appStates, appTags);
- } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
- if (args.length != 3) {
+ } else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) {
printUsage(title, opts);
return exitCode;
}
listApplicationAttempts(cliParser.getOptionValue(LIST_CMD));
- } else if (args[0].equalsIgnoreCase(CONTAINER)) {
- if (args.length != 3) {
+ } else if (title.equalsIgnoreCase(CONTAINER)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) {
printUsage(title, opts);
return exitCode;
}
listContainers(cliParser.getOptionValue(LIST_CMD));
}
} else if (cliParser.hasOption(KILL_CMD)) {
- if (args.length < 3 || hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
printUsage(title, opts);
return exitCode;
}
@@ -299,7 +399,7 @@ public class ApplicationCLI extends YarnCLI {
moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD),
cliParser.getOptionValue(QUEUE_CMD));
} else if (cliParser.hasOption(FAIL_CMD)) {
- if (!args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
+ if (!title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
printUsage(title, opts);
return exitCode;
}
@@ -314,6 +414,103 @@ public class ApplicationCLI extends YarnCLI {
}
updateApplicationPriority(cliParser.getOptionValue(APP_ID),
cliParser.getOptionValue(UPDATE_PRIORITY));
+ } else if (cliParser.hasOption(SIGNAL_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, SIGNAL_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD);
+ final String containerId = signalArgs[0];
+ SignalContainerCommand command =
+ SignalContainerCommand.OUTPUT_THREAD_DUMP;
+ if (signalArgs.length == 2) {
+ command = SignalContainerCommand.valueOf(signalArgs[1]);
+ }
+ signalToContainer(containerId, command);
+ } else if (cliParser.hasOption(LAUNCH_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, LAUNCH_CMD, APP_TYPE_CMD,
+ UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ Long lifetime = null;
+ if (cliParser.hasOption(UPDATE_LIFETIME)) {
+ lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
+ }
+ String queue = null;
+ if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) {
+ queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE);
+ }
+ String[] nameAndFile = cliParser.getOptionValues(LAUNCH_CMD);
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .actionLaunch(nameAndFile[1], nameAndFile[0], lifetime, queue);
+ } else if (cliParser.hasOption(STOP_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, STOP_CMD, APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String[] appNameAndType = getAppNameAndType(cliParser, STOP_CMD);
+ return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf())
+ .actionStop(appNameAndType[0]);
+ } else if (cliParser.hasOption(START_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, START_CMD, APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .actionStart(cliParser.getOptionValue(START_CMD));
+ } else if (cliParser.hasOption(SAVE_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, SAVE_CMD, APP_TYPE_CMD,
+ UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ Long lifetime = null;
+ if (cliParser.hasOption(UPDATE_LIFETIME)) {
+ lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
+ }
+ String queue = null;
+ if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) {
+ queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE);
+ }
+ String[] nameAndFile = cliParser.getOptionValues(SAVE_CMD);
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .actionSave(nameAndFile[1], nameAndFile[0], lifetime, queue);
+ } else if (cliParser.hasOption(DESTROY_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, DESTROY_CMD, APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .actionDestroy(cliParser.getOptionValue(DESTROY_CMD));
+ } else if (cliParser.hasOption(FLEX_CMD)) {
+ if (!cliParser.hasOption(COMPONENT) ||
+ hasAnyOtherCLIOptions(cliParser, opts, FLEX_CMD, COMPONENT,
+ APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String[] rawCounts = cliParser.getOptionValues(COMPONENT);
+ Map<String, String> counts = new HashMap<>(rawCounts.length/2);
+ for (int i = 0; i < rawCounts.length - 1; i+=2) {
+ counts.put(rawCounts[i], rawCounts[i+1]);
+ }
+ String[] appNameAndType = getAppNameAndType(cliParser, FLEX_CMD);
+ return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf())
+ .actionFlex(appNameAndType[0], counts);
+ } else if (cliParser.hasOption(ENABLE_FAST_LAUNCH)) {
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ if (hasAnyOtherCLIOptions(cliParser, opts, ENABLE_FAST_LAUNCH,
+ APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .enableFastLaunch();
} else if (cliParser.hasOption(UPDATE_LIFETIME)) {
if (!cliParser.hasOption(APP_ID)) {
printUsage(title, opts);
@@ -332,19 +529,6 @@ public class ApplicationCLI extends YarnCLI {
}
moveApplicationAcrossQueues(cliParser.getOptionValue(APP_ID),
cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE));
- } else if (cliParser.hasOption(SIGNAL_CMD)) {
- if (args.length < 3 || args.length > 4) {
- printUsage(title, opts);
- return exitCode;
- }
- final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD);
- final String containerId = signalArgs[0];
- SignalContainerCommand command =
- SignalContainerCommand.OUTPUT_THREAD_DUMP;
- if (signalArgs.length == 2) {
- command = SignalContainerCommand.valueOf(signalArgs[1]);
- }
- signalToContainer(containerId, command);
} else {
syserr.println("Invalid Command Usage : ");
printUsage(title, opts);
@@ -352,6 +536,47 @@ public class ApplicationCLI extends YarnCLI {
return 0;
}
+ private ApplicationReport getApplicationReport(ApplicationId applicationId)
+ throws IOException, YarnException {
+ ApplicationReport appReport = null;
+ try {
+ appReport = client.getApplicationReport(applicationId);
+ } catch (ApplicationNotFoundException e) {
+ throw new YarnException("Application with id '" + applicationId
+ + "' doesn't exist in RM or Timeline Server.");
+ }
+ return appReport;
+ }
+
+ private String[] getAppNameAndType(CommandLine cliParser, String option)
+ throws IOException, YarnException {
+ String applicationIdOrName = cliParser.getOptionValue(option);
+ try {
+ ApplicationId id = ApplicationId.fromString(applicationIdOrName);
+ ApplicationReport report = getApplicationReport(id);
+ return new String[]{report.getName(), report.getApplicationType()};
+ } catch (IllegalArgumentException e) {
+ // assume CLI option provided the app name
+ // and read appType from command line since id wasn't provided
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ return new String[]{applicationIdOrName, appType};
+ }
+ }
+
+ private static String getSingleAppTypeFromCLI(CommandLine cliParser) {
+ if (cliParser.hasOption(APP_TYPE_CMD)) {
+ String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
+ if (types != null) {
+ for (String type : types) {
+ if (!type.trim().isEmpty()) {
+ return StringUtils.toLowerCase(type).trim();
+ }
+ }
+ }
+ }
+ return AppAdminClient.DEFAULT_TYPE;
+ }
+
private void updateApplicationTimeout(String applicationId,
ApplicationTimeoutType timeoutType, long timeoutInSec)
throws YarnException, IOException {
@@ -572,7 +797,7 @@ public class ApplicationCLI extends YarnCLI {
/**
* Kills applications with the application id as appId
*
- * @param Array of applicationIds
+ * @param applicationIds Array of applicationIds
* @return errorCode
* @throws YarnException
* @throws IOException
@@ -663,10 +888,10 @@ public class ApplicationCLI extends YarnCLI {
* Prints the application report for an application id.
*
* @param applicationId
- * @return exitCode
+ * @return ApplicationReport
* @throws YarnException
*/
- private int printApplicationReport(String applicationId)
+ private ApplicationReport printApplicationReport(String applicationId)
throws YarnException, IOException {
ApplicationReport appReport = null;
try {
@@ -675,7 +900,7 @@ public class ApplicationCLI extends YarnCLI {
} catch (ApplicationNotFoundException e) {
sysout.println("Application with id '" + applicationId
+ "' doesn't exist in RM or Timeline Server.");
- return -1;
+ return null;
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -739,11 +964,11 @@ public class ApplicationCLI extends YarnCLI {
+ "' doesn't exist in RM.");
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
- return -1;
+ return null;
}
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
- return 0;
+ return appReport;
}
private void printResourceUsage(PrintWriter appReportStr,
@@ -856,11 +1081,12 @@ public class ApplicationCLI extends YarnCLI {
@SuppressWarnings("unchecked")
private boolean hasAnyOtherCLIOptions(CommandLine cliParser, Options opts,
- String excludeOption) {
+ String... excludeOptions) {
Collection<Option> ops = opts.getOptions();
+ Set<String> excludeSet = new HashSet<>(Arrays.asList(excludeOptions));
for (Option op : ops) {
- // Skip exclude option from the option list
- if (op.getOpt().equals(excludeOption)) {
+ // Skip exclude options from the option list
+ if (excludeSet.contains(op.getOpt())) {
continue;
}
if (cliParser.hasOption(op.getOpt())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 84cfb0a..da1659f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -2025,49 +2025,121 @@ public class TestYarnCLI {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("usage: application");
- pw.println(" -appId <Application ID> Specify Application Id to be operated");
- pw.println(" -appStates <States> Works with -list to filter applications");
- pw.println(" based on input comma-separated list of");
- pw.println(" application states. The valid application");
- pw.println(" state can be one of the following:");
- pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN");
- pw.println(" NING,FINISHED,FAILED,KILLED");
- pw.println(" -appTags <Tags> Works with -list to filter applications");
- pw.println(" based on input comma-separated list of");
- pw.println(" application tags.");
- pw.println(" -appTypes <Types> Works with -list to filter applications");
- pw.println(" based on input comma-separated list of");
- pw.println(" application types.");
- pw.println(" -changeQueue <Queue Name> Moves application to a new queue.");
- pw.println(" ApplicationId can be passed using 'appId'");
- pw.println(" option. 'movetoqueue' command is");
- pw.println(" deprecated, this new command");
- pw.println(" 'changeQueue' performs same");
- pw.println(" functionality.");
- pw.println(" -help Displays help for all commands.");
- pw.println(" -kill <Application ID> Kills the application. Set of");
- pw.println(" applications can be provided separated");
- pw.println(" with space");
- pw.println(" -list List applications. Supports optional use");
- pw.println(" of -appTypes to filter applications based");
- pw.println(" on application type, -appStates to filter");
- pw.println(" applications based on application state");
- pw.println(" and -appTags to filter applications based");
- pw.println(" on application tag.");
- pw.println(" -movetoqueue <Application ID> Moves the application to a different");
- pw.println(" queue. Deprecated command. Use");
- pw.println(" 'changeQueue' instead.");
- pw.println(" -queue <Queue Name> Works with the movetoqueue command to");
- pw.println(" specify which queue to move an");
- pw.println(" application to.");
- pw.println(" -status <Application ID> Prints the status of the application.");
- pw.println(" -updateLifetime <Timeout> update timeout of an application from");
- pw.println(" NOW. ApplicationId can be passed using");
- pw.println(" 'appId' option. Timeout value is in");
- pw.println(" seconds.");
- pw.println(" -updatePriority <Priority> update priority of an application.");
- pw.println(" ApplicationId can be passed using 'appId'");
- pw.println(" option.");
+ pw.println(" -appId <Application ID> Specify Application Id to be");
+ pw.println(" operated");
+ pw.println(" -appStates <States> Works with -list to filter");
+ pw.println(" applications based on input");
+ pw.println(" comma-separated list of");
+ pw.println(" application states. The valid");
+ pw.println(" application state can be one of");
+ pw.println(" the following:");
+ pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACC");
+ pw.println(" EPTED,RUNNING,FINISHED,FAILED,KI");
+ pw.println(" LLED");
+ pw.println(" -appTags <Tags> Works with -list to filter");
+ pw.println(" applications based on input");
+ pw.println(" comma-separated list of");
+ pw.println(" application tags.");
+ pw.println(" -appTypes <Types> Works with -list to filter");
+ pw.println(" applications based on input");
+ pw.println(" comma-separated list of");
+ pw.println(" application types.");
+ pw.println(" -changeQueue <Queue Name> Moves application to a new");
+ pw.println(" queue. ApplicationId can be");
+ pw.println(" passed using 'appId' option.");
+ pw.println(" 'movetoqueue' command is");
+ pw.println(" deprecated, this new command");
+ pw.println(" 'changeQueue' performs same");
+ pw.println(" functionality.");
+ pw.println(" -component <Component Name> <Count> Works with -flex option to");
+ pw.println(" change the number of");
+ pw.println(" components/containers running");
+ pw.println(" for an application /");
+ pw.println(" long-running service. Supports");
+ pw.println(" absolute or relative changes,");
+ pw.println(" such as +1, 2, or -3.");
+ pw.println(" -destroy <Application Name> Destroys a saved application");
+ pw.println(" specification and removes all");
+ pw.println(" application data permanently.");
+ pw.println(" Supports -appTypes option to");
+ pw.println(" specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -enableFastLaunch Uploads AM dependencies to HDFS");
+ pw.println(" to make future launches faster.");
+ pw.println(" Supports -appTypes option to");
+ pw.println(" specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -flex <Application Name or ID> Changes number of running");
+ pw.println(" containers for a component of an");
+ pw.println(" application / long-running");
+ pw.println(" service. Requires -component");
+ pw.println(" option. If name is provided,");
+ pw.println(" appType must be provided unless");
+ pw.println(" it is the default yarn-service.");
+ pw.println(" If ID is provided, the appType");
+ pw.println(" will be looked up. Supports");
+ pw.println(" -appTypes option to specify");
+ pw.println(" which client implementation to");
+ pw.println(" use.");
+ pw.println(" -help Displays help for all commands.");
+ pw.println(" -kill <Application ID> Kills the application. Set of");
+ pw.println(" applications can be provided");
+ pw.println(" separated with space");
+ pw.println(" -launch <Application Name> <File Name> Launches application from");
+ pw.println(" specification file (saves");
+ pw.println(" specification and starts");
+ pw.println(" application). Options");
+ pw.println(" -updateLifetime and -changeQueue");
+ pw.println(" can be specified to alter the");
+ pw.println(" values provided in the file.");
+ pw.println(" Supports -appTypes option to");
+ pw.println(" specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -list List applications. Supports");
+ pw.println(" optional use of -appTypes to");
+ pw.println(" filter applications based on");
+ pw.println(" application type, -appStates to");
+ pw.println(" filter applications based on");
+ pw.println(" application state and -appTags");
+ pw.println(" to filter applications based on");
+ pw.println(" application tag.");
+ pw.println(" -movetoqueue <Application ID> Moves the application to a");
+ pw.println(" different queue. Deprecated");
+ pw.println(" command. Use 'changeQueue'");
+ pw.println(" instead.");
+ pw.println(" -queue <Queue Name> Works with the movetoqueue");
+ pw.println(" command to specify which queue");
+ pw.println(" to move an application to.");
+ pw.println(" -save <Application Name> <File Name> Saves specification file for an");
+ pw.println(" application. Options");
+ pw.println(" -updateLifetime and -changeQueue");
+ pw.println(" can be specified to alter the");
+ pw.println(" values provided in the file.");
+ pw.println(" Supports -appTypes option to");
+ pw.println(" specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -start <Application Name> Starts a previously saved");
+ pw.println(" application. Supports -appTypes");
+ pw.println(" option to specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -status <Application ID> Prints the status of the");
+ pw.println(" application.");
+ pw.println(" -stop <Application Name or ID> Stops application gracefully");
+ pw.println(" (may be started again later). If");
+ pw.println(" name is provided, appType must");
+ pw.println(" be provided unless it is the");
+ pw.println(" default yarn-service. If ID is");
+ pw.println(" provided, the appType will be");
+ pw.println(" looked up. Supports -appTypes");
+ pw.println(" option to specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -updateLifetime <Timeout> update timeout of an application");
+ pw.println(" from NOW. ApplicationId can be");
+ pw.println(" passed using 'appId' option.");
+ pw.println(" Timeout value is in seconds.");
+ pw.println(" -updatePriority <Priority> update priority of an");
+ pw.println(" application. ApplicationId can");
+ pw.println(" be passed using 'appId' option.");
pw.close();
String appsHelpStr = baos.toString("UTF-8");
return appsHelpStr;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 6f00c5c..71c9e80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -37,25 +37,36 @@ User Commands
Commands useful for users of a Hadoop cluster.
-### `application`
+### `application` or `app`
Usage: `yarn application [options] `
+Usage: `yarn app [options] `
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| -appId \<ApplicationId\> | Specify Application Id to be operated |
| -appStates \<States\> | Works with -list to filter applications based on input comma-separated list of application states. The valid application state can be one of the following: ALL, NEW, NEW\_SAVING, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED |
+| -appTags \<Tags\> | Works with -list to filter applications based on input comma-separated list of application tags. |
| -appTypes \<Types\> | Works with -list to filter applications based on input comma-separated list of application types. |
-| -help | Help |
-| -list | Lists applications from the RM. Supports optional use of -appTypes to filter applications based on application type, and -appStates to filter applications based on application state. |
-| -kill \<ApplicationId\> | Kills the application. |
-| -movetoqueue \<Application Id\> | Moves the application to a different queue. |
+| -changeQueue \<Queue Name\> | Moves application to a new queue. ApplicationId can be passed using 'appId' option. 'movetoqueue' command is deprecated, this new command 'changeQueue' performs same functionality. |
+| -component \<Component Name\> \<Count\> | Works with -flex option to change the number of components/containers running for an application / long-running service. Supports absolute or relative changes, such as +1, 2, or -3. |
+| -destroy \<Application Name\> | Destroys a saved application specification and removes all application data permanently. Supports -appTypes option to specify which client implementation to use. |
+| -enableFastLaunch | Uploads AM dependencies to HDFS to make future launches faster. Supports -appTypes option to specify which client implementation to use. |
+| -flex \<Application Name or ID\> | Changes number of running containers for a component of an application / long-running service. Requires -component option. If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. |
+| -help | Displays help for all commands. |
+| -kill \<Application ID\> | Kills the application. Set of applications can be provided separated with space |
+| -launch \<Application Name\> \<File Name\> | Launches application from specification file (saves specification and starts application). Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. |
+| -list | List applications. Supports optional use of -appTypes to filter applications based on application type, -appStates to filter applications based on application state and -appTags to filter applications based on application tag. |
+| -movetoqueue \<Application ID\> | Moves the application to a different queue. Deprecated command. Use 'changeQueue' instead. |
| -queue \<Queue Name\> | Works with the movetoqueue command to specify which queue to move an application to. |
+| -save \<Application Name\> \<File Name\> | Saves specification file for an application. Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. |
+| -start \<Application Name\> | Starts a previously saved application. Supports -appTypes option to specify which client implementation to use. |
| -status \<ApplicationId\> | Prints the status of the application. |
-| -updateLifetime \<Timeout\> | Update application timeout (from the time of request) in seconds. ApplicationId can be specified using 'appId' option. |
+| -stop \<Application Name or ID\> | Stops application gracefully (may be started again later). If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. |
+| -updateLifetime \<Timeout\> | Update timeout of an application from NOW. ApplicationId can be passed using 'appId' option. Timeout value is in seconds. |
| -updatePriority \<Priority\> | Update priority of an application. ApplicationId can be passed using 'appId' option. |
-Prints application(s) report/kill application
+Prints application(s) report/kill application/manage long running application
### `applicationattempt`
@@ -69,61 +80,6 @@ Usage: `yarn applicationattempt [options] `
prints applicationattempt(s) report
-### `service`
-Usage `yarn service [sub-command] [service-name] [options]`
-
-#### `SUB_COMMAND`
-
-* `build`: Build a service with its specifications, but do not start it.
- ```
- Usage: yarn service build [service-name] --file [file]
- Fields:
- service-name Optional. If specified, it will override the name in the service definition.
-
- Options:
- --file,-f The local path to the service definition file
- ```
-
-* `create`: create a service, it's equivalent to first invoke build and then start.
- ```
- Usage: yarn service create [service-name] --file [file]
- Fields:
- service-name Optional. If specified, it will override the name in the service definition.
-
- Options:
- --file,-f The local path to the service definition file.
- --queue,-q The queue to which the service is submitted.
- --example,-e The name of the example service such as:
- Sleeper A simple service that launches a few non-docker sleep containers on YARN.
- ```
-
-* `dependency`: Yarn service framework dependency (libraries) management.
- ```
- Usage: yarn service dependency [options]
- Option:
- --upload Pre-upload the dependency jars onto HDFS to expediate service launch process.
- ```
-
-* `destroy`: Destroy a stopped service, service must be stopped first before destroying.
- ```
- Usage: yarn service destroy [service-name]
- ```
-* `flex`: Flex a service's component by increasing or decreasing the number of containers.
- ```
- Usage: yarn service flex [service-name] --component [component-name] [count]
- Options:
- --component [component-name] [count]
- Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.
- ```
-* `status`: Get the status of a service.
- ```
- Usage: yarn service status [service-name]
- ```
-* `start`: Start a service with pre-built specification or a previously stopped service.
- ```
- Usage: yarn service start [service-name]
- ```
-
### `classpath`
Usage: `yarn classpath [--glob |--jar <path> |-h |--help]`
@@ -328,12 +284,6 @@ Usage: `yarn timelineserver`
Start the TimeLineServer
-### apiserver
-
-Usage: `yarn apiserver`
-
-Start the API-server for deploying/managing services on YARN
-
### registrydns
Usage: `yarn registrydns`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
index db6d553..e4881ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -100,16 +100,18 @@ ProxyPassReverse "/" "balancer://test/"
Then run the service with the command:
```
-yarn service create [service-name] --example httpd
+yarn app -launch <service-name> httpd
```
-where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
+
+The last argument is either the path to a JSON specification of the service, or in this case, the name of an example service.
+The directory where examples can be found can be configured by setting the YARN\_EXAMPLES\_DIR environment variable.
Once the service is running, navigate to `http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` to see the root page.
The pages should alternately show "Hello from httpd-0!" or "Hello from httpd-1!"
The individual httpd URLs can also be visited, `http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` and `http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080`.
-If unsure of your hostnames, visit the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
+If unsure of your hostnames, visit the RM REST endpoint `http://<RM host>:8088/ws/v1/services/httpd-service`.
## Apache web server - httpd (without registry DNS)
@@ -151,9 +153,9 @@ hdfs dfs -copyFromLocal ${HADOOP_YARN_HOME}/share/hadoop/yarn/yarn-service-examp
Then run the service with the command:
```
-yarn service create [service-name] --example httpd-no-dns
+yarn app -launch <service-name> httpd-no-dns
```
where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
-Look up your IPs at the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
+Look up your IPs at the RM REST endpoint `http://<RM host>:8088/ws/v1/services/httpd-service`.
Then visit port 8080 for each IP to view the pages.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index afcd5f5..512c011 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -45,11 +45,11 @@ Below is a simple service definition that launches sleep containers on YARN by w
```
User can simply run a pre-built example service on YARN using below command:
```
-yarn service create [service-name] --example [example-name]
+yarn app -launch <service-name> <example-name>
```
e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN.
```
-yarn service create my-sleeper --example sleeper
+yarn app -launch my-sleeper sleeper
```
For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
@@ -58,51 +58,53 @@ Below steps walk you through deploying a services on YARN using CLI.
Refer to [Yarn Commands](../YarnCommands.md) for the full list of commands and options.
### Deploy a service
```
-yarn service create --file ${PATH_TO_SERVICE_DEF_FILE}
+yarn app -launch ${SERVICE_NAME} ${PATH_TO_SERVICE_DEF_FILE}
```
Params:
-- SERVICE_NAME: The name of the service. Note that this needs to be unique across all running services.
+- SERVICE_NAME: The name of the service. Note that this needs to be unique across running services for the current user.
- PATH_TO_SERVICE_DEF: The path to the service definition file in JSON format.
For example:
```
-yarn service create --file /path/to/local/sleeper.json
+yarn app -launch sleeper-service /path/to/local/sleeper.json
```
### Flex a component of a service
Increase or decrease the number of containers for a component.
```
-yarn service flex ${SERVICE_NAME} --component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS}
+yarn app -flex ${SERVICE_NAME} -component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS}
```
For example, for a service named `sleeper-service`:
Set the `sleeper` component to `2` containers (absolute number).
```
-yarn service flex sleeper-service --component sleeper 2
+yarn app -flex sleeper-service -component sleeper 2
```
+Relative changes are also supported for the ${NUMBER_OF_CONTAINERS} in the flex command, such as +2 or -2.
+
### Stop a service
Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
```
-yarn service stop ${SERVICE_NAME}
+yarn app -stop ${SERVICE_NAME}
```
### Restart a stopped service
Restarting a stopped service is easy - just call start!
```
-yarn service start ${SERVICE_NAME}
+yarn app -start ${SERVICE_NAME}
```
### Destroy a service
In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
```
-yarn service destroy ${SERVICE_NAME}
+yarn app -destroy ${SERVICE_NAME}
```
## Manage services on YARN via REST API
-YARN API Server REST API can be activated in two modes: embedded or standalone.
+YARN API Server REST API can be activated as part of the ResourceManager.
### Start Embedded API-Server as part of ResourceManager
For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
@@ -119,13 +121,6 @@ For running inside ResourceManager, add this property to `yarn-site.xml` and res
Services can be deployed on YARN through the ResourceManager web endpoint.
-### Start Standalone API-Server for deploying services on YARN
-API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP.
-```
- yarn --daemon start apiserver
- ```
-The above command starts the API Server on the localhost at port 9191 by default.
-
Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
### Deploy a service
@@ -135,19 +130,14 @@ POST the aforementioned example service definition to the ResourceManager api-se
POST http://localhost:8088/ws/v1/services
```
-Or standalone API server:
-```
-POST http://localhost:9191/ws/v1/services
-```
-
### Get a service status
```
-GET http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+GET http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
### Flex a component of a service
```
-PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME}
+PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME}
```
`PUT` Request body:
```
@@ -168,7 +158,7 @@ For example:
Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
```
-PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
`PUT` Request body:
@@ -183,7 +173,7 @@ PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
Restarting a stopped service is easy.
```
-PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
`PUT` Request body:
@@ -196,7 +186,7 @@ PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
### Destroy a service
In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
```
-DELETE http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+DELETE http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
## Services UI with YARN UI2 and Timeline Service v2
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index e9aaf7e..7a2490e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -37,7 +37,7 @@ License URL: http://www.apache.org/licenses/LICENSE-2.0.html
### URI scheme
Host: host.mycompany.com
-Port: 9191(default)
+Port: 8088(default RM port)
Schemes: HTTP
@@ -368,7 +368,7 @@ The current status of a submitted service, returned as a response to the GET API
## Examples
### Create a simple single-component service with most attribute values as defaults
-POST URL - http://localhost:9191/ws/v1/services
+POST URL - http://localhost:8088/ws/v1/services
##### POST Request JSON
```json
@@ -394,7 +394,7 @@ POST URL - http://localhost:9191/ws/v1/services
```
##### GET Response JSON
-GET URL - http://localhost:9191/ws/v1/services/hello-world
+GET URL - http://localhost:8088/ws/v1/services/hello-world
Note, lifetime value of -1 means unlimited lifetime.
@@ -452,7 +452,7 @@ Note, lifetime value of -1 means unlimited lifetime.
```
### Update to modify the lifetime of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/ws/v1/services/hello-world
##### PUT Request JSON
@@ -464,7 +464,7 @@ Note, irrespective of what the current lifetime value is, this update request wi
}
```
### Stop a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/ws/v1/services/hello-world
##### PUT Request JSON
```json
@@ -474,7 +474,7 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world
```
### Start a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/ws/v1/services/hello-world
##### PUT Request JSON
```json
@@ -484,7 +484,7 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world
```
### Update to flex up/down the no of containers (instances) of a component of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
+PUT URL - http://localhost:8088/ws/v1/services/hello-world/components/hello
##### PUT Request JSON
```json
@@ -495,12 +495,12 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
```
### Destroy a service
-DELETE URL - http://localhost:9191/ws/v1/services/hello-world
+DELETE URL - http://localhost:8088/ws/v1/services/hello-world
***
### Create a complicated service - HBase
-POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
+POST URL - http://localhost:8088:/ws/v1/services/hbase-app-1
##### POST Request JSON
@@ -595,4 +595,4 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
"Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
}
}
-```
\ No newline at end of file
+```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
deleted file mode 100644
index 1049698..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ /dev/null
@@ -1,872 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.RetryNTimes;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.ServiceMaster;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.client.params.CommonArgs;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
-import org.apache.hadoop.yarn.service.provider.ProviderUtils;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
-import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
-import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.text.MessageFormat;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
-import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class ServiceClient extends CompositeService
- implements SliderExitCodes, YarnServiceConstants {
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceClient.class);
- private SliderFileSystem fs;
- private YarnClient yarnClient;
- // Avoid looking up applicationId from fs all the time.
- private Map<String, ApplicationId> cachedAppIds = new ConcurrentHashMap<>();
- private Map<String, ClientAMProtocol> cachedAMProxies = new ConcurrentHashMap<>();
-
- private RegistryOperations registryClient;
- private CuratorFramework curatorClient;
- private YarnRPC rpc;
-
- private static EnumSet<YarnApplicationState> terminatedStates =
- EnumSet.of(FINISHED, FAILED, KILLED);
- private static EnumSet<YarnApplicationState> liveStates =
- EnumSet.of(NEW, NEW_SAVING, SUBMITTED, RUNNING);
-
- public ServiceClient() {
- super(ServiceClient.class.getName());
- }
-
- @Override protected void serviceInit(Configuration configuration)
- throws Exception {
- fs = new SliderFileSystem(configuration);
- yarnClient = YarnClient.createYarnClient();
- rpc = YarnRPC.create(configuration);
- addService(yarnClient);
- super.serviceInit(configuration);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (registryClient != null) {
- registryClient.stop();
- }
- super.serviceStop();
- }
-
- private Application loadAppJsonFromLocalFS(
- AbstractClusterBuildingActionArgs args) throws IOException {
- File file = args.getAppDef();
- Path filePath = new Path(file.getAbsolutePath());
- LOG.info("Loading app json from: " + filePath);
- Application application = ServiceApiUtil.jsonSerDeser
- .load(FileSystem.getLocal(getConfig()), filePath);
- if (args.lifetime > 0) {
- application.setLifetime(args.lifetime);
- }
- application.setName(args.getClusterName());
- return application;
- }
-
- public int actionBuild(AbstractClusterBuildingActionArgs args)
- throws IOException, YarnException {
- return actionBuild(loadAppJsonFromLocalFS(args));
- }
-
- public int actionBuild(Application application)
- throws YarnException, IOException {
- Path appDir = checkAppNotExistOnHdfs(application);
- ServiceApiUtil.validateAndResolveApplication(application, fs, getConfig());
- createDirAndPersistApp(appDir, application);
- return EXIT_SUCCESS;
- }
-
- public int actionCreate(AbstractClusterBuildingActionArgs args)
- throws IOException, YarnException {
- actionCreate(loadAppJsonFromLocalFS(args));
- return EXIT_SUCCESS;
- }
-
- public ApplicationId actionCreate(Application application)
- throws IOException, YarnException {
- String appName = application.getName();
- validateClusterName(appName);
- ServiceApiUtil.validateAndResolveApplication(application, fs, getConfig());
- verifyNoLiveAppInRM(appName, "create");
- Path appDir = checkAppNotExistOnHdfs(application);
-
- // Write the definition first and then submit - AM will read the definition
- createDirAndPersistApp(appDir, application);
- ApplicationId appId = submitApp(application);
- cachedAppIds.put(appName, appId);
- application.setId(appId.toString());
- // update app definition with appId
- persistAppDef(appDir, application);
- return appId;
- }
-
- // Called by ServiceCLI
- protected int actionFlexByCLI(ClientArgs args)
- throws YarnException, IOException {
- ActionFlexArgs flexArgs = args.getActionFlexArgs();
- Map<String, Long> componentCounts =
- new HashMap<>(flexArgs.getComponentMap().size());
- Application persistedApp =
- ServiceApiUtil.loadApplication(fs, flexArgs.getClusterName());
- if (!StringUtils.isEmpty(persistedApp.getId())) {
- cachedAppIds.put(persistedApp.getName(),
- ApplicationId.fromString(persistedApp.getId()));
- }
- for (Map.Entry<String, String> entry : flexArgs.getComponentMap()
- .entrySet()) {
- String compName = entry.getKey();
- ServiceApiUtil.validateCompName(compName);
- Component component = persistedApp.getComponent(compName);
- if (component == null) {
- throw new IllegalArgumentException(entry.getKey() + " does not exist !");
- }
- long numberOfContainers =
- parseNumberOfContainers(component, entry.getValue());
- componentCounts.put(compName, numberOfContainers);
- }
- // throw usage exception if no changes proposed
- if (componentCounts.size() == 0) {
- actionHelp(ACTION_FLEX, args);
- }
- flexComponents(args.getClusterName(), componentCounts, persistedApp);
- return EXIT_SUCCESS;
- }
-
- // Parse the number of containers requested by user, e.g.
- // +5 means add 5 additional containers
- // -5 means reduce 5 containers, if it goes to negative, sets it to 0
- // 5 means sets it to 5 containers.
- private long parseNumberOfContainers(Component component, String newNumber) {
-
- long orig = component.getNumberOfContainers();
- if (newNumber.startsWith("+")) {
- return orig + Long.parseLong(newNumber.substring(1));
- } else if (newNumber.startsWith("-")) {
- long ret = orig - Long.parseLong(newNumber.substring(1));
- if (ret < 0) {
- LOG.warn(MessageFormat.format(
- "[COMPONENT {}]: component count goes to negative ({}{} = {}), reset it to 0.",
- component.getName(), orig, newNumber, ret));
- ret = 0;
- }
- return ret;
- } else {
- return Long.parseLong(newNumber);
- }
- }
-
- // Called by Rest Service
- public Map<String, Long> flexByRestService(String appName,
- Map<String, Long> componentCounts) throws YarnException, IOException {
- // load app definition
- Application persistedApp = ServiceApiUtil.loadApplication(fs, appName);
- cachedAppIds.put(persistedApp.getName(),
- ApplicationId.fromString(persistedApp.getId()));
- return flexComponents(appName, componentCounts, persistedApp);
- }
-
- private Map<String, Long> flexComponents(String appName,
- Map<String, Long> componentCounts, Application persistedApp)
- throws YarnException, IOException {
- validateClusterName(appName);
-
- Map<String, Long> original = new HashMap<>(componentCounts.size());
-
- ComponentCountProto.Builder countBuilder = ComponentCountProto.newBuilder();
- FlexComponentsRequestProto.Builder requestBuilder =
- FlexComponentsRequestProto.newBuilder();
-
- for (Component persistedComp : persistedApp.getComponents()) {
- String name = persistedComp.getName();
- if (componentCounts.containsKey(persistedComp.getName())) {
- original.put(name, persistedComp.getNumberOfContainers());
- persistedComp.setNumberOfContainers(componentCounts.get(name));
-
- // build the request
- countBuilder.setName(persistedComp.getName())
- .setNumberOfContainers(persistedComp.getNumberOfContainers());
- requestBuilder.addComponents(countBuilder.build());
- }
- }
- if (original.size() < componentCounts.size()) {
- componentCounts.keySet().removeAll(original.keySet());
- throw new YarnException("Components " + componentCounts.keySet()
- + " do not exist in app definition.");
- }
- ServiceApiUtil.jsonSerDeser
- .save(fs.getFileSystem(), ServiceApiUtil.getAppJsonPath(fs, appName),
- persistedApp, true);
- ClientAMProtocol proxy = getAMProxy(appName);
- if (proxy == null) {
- String message = appName + " is not running";
- LOG.error(message);
- throw new YarnException(message);
- }
- proxy.flexComponents(requestBuilder.build());
- for (Map.Entry<String, Long> entry : original.entrySet()) {
- LOG.info("[COMPONENT {}]: number of containers changed from {} to {}",
- entry.getKey(), entry.getValue(),
- componentCounts.get(entry.getKey()));
- }
- return original;
- }
-
- public int actionStop(String appName, boolean waitForAppStopped)
- throws YarnException, IOException {
- validateClusterName(appName);
- getAppId(appName);
- ApplicationId currentAppId = cachedAppIds.get(appName);
- ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
- if (terminatedStates.contains(report.getYarnApplicationState())) {
- LOG.info("Application {} is already in a terminated state {}", appName,
- report.getYarnApplicationState());
- return EXIT_SUCCESS;
- }
- LOG.info("Stopping application {}, with appId = {}", appName, currentAppId);
- try {
- ClientAMProtocol proxy = getAMProxy(appName, report);
- cachedAppIds.remove(appName);
- cachedAMProxies.remove(appName);
- if (proxy != null) {
- // try to stop the app gracefully.
- StopRequestProto request = StopRequestProto.newBuilder().build();
- proxy.stop(request);
- LOG.info("Application " + appName + " is being gracefully stopped...");
- } else {
- yarnClient.killApplication(currentAppId,
- appName + " is forcefully killed by user!");
- LOG.info("Forcefully kill the application: " + appName);
- return EXIT_SUCCESS;
- }
-
- if (!waitForAppStopped) {
- return EXIT_SUCCESS;
- }
- // Wait until the app is killed.
- long startTime = System.currentTimeMillis();
- int pollCount = 0;
- while (true) {
- Thread.sleep(2000);
- report = yarnClient.getApplicationReport(currentAppId);
- if (terminatedStates.contains(report.getYarnApplicationState())) {
- LOG.info("Application " + appName + " is stopped.");
- break;
- }
- // Forcefully kill after 10 seconds.
- if ((System.currentTimeMillis() - startTime) > 10000) {
- LOG.info("Stop operation timeout stopping, forcefully kill the app "
- + appName);
- yarnClient.killApplication(currentAppId,
- "Forcefully kill the app by user");
- break;
- }
- if (++pollCount % 10 == 0) {
- LOG.info("Waiting for application " + appName + " to be stopped.");
- }
- }
- } catch (IOException | YarnException | InterruptedException e) {
- LOG.info("Failed to stop " + appName
- + " gracefully, forcefully kill the app.");
- yarnClient.killApplication(currentAppId, "Forcefully kill the app");
- }
- return EXIT_SUCCESS;
- }
-
- public int actionDestroy(String appName) throws Exception {
- validateClusterName(appName);
- verifyNoLiveAppInRM(appName, "Destroy");
- Path appDir = fs.buildClusterDirPath(appName);
- FileSystem fileSystem = fs.getFileSystem();
- // remove from the appId cache
- cachedAppIds.remove(appName);
- cachedAMProxies.remove(appName);
- if (fileSystem.exists(appDir)) {
- if (fileSystem.delete(appDir, true)) {
- LOG.info("Successfully deleted application dir for " + appName + ": "
- + appDir);
- } else {
- String message =
- "Failed to delete application + " + appName + " at: " + appDir;
- LOG.info(message);
- throw new YarnException(message);
- }
- }
- deleteZKNode(appName);
- String registryPath = ServiceRegistryUtils.registryPathForInstance(appName);
- try {
- getRegistryClient().delete(registryPath, true);
- } catch (IOException e) {
- LOG.warn("Error deleting registry entry {}", registryPath, e);
- }
- LOG.info("Destroyed cluster {}", appName);
- return EXIT_SUCCESS;
- }
-
- private synchronized RegistryOperations getRegistryClient()
- throws SliderException, IOException {
-
- if (registryClient == null) {
- registryClient =
- RegistryOperationsFactory.createInstance("ServiceClient", getConfig());
- registryClient.init(getConfig());
- registryClient.start();
- }
- return registryClient;
- }
-
- private void deleteZKNode(String clusterName) throws Exception {
- CuratorFramework curatorFramework = getCuratorClient();
- String user = RegistryUtils.currentUser();
- String zkPath = ServiceRegistryUtils.mkClusterPath(user, clusterName);
- if (curatorFramework.checkExists().forPath(zkPath) != null) {
- curatorFramework.delete().deletingChildrenIfNeeded().forPath(zkPath);
- LOG.info("Deleted zookeeper path: " + zkPath);
- }
- }
-
- private synchronized CuratorFramework getCuratorClient()
- throws BadConfigException {
- String registryQuorum =
- getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
-
- // though if neither is set: trouble
- if (SliderUtils.isUnset(registryQuorum)) {
- throw new BadConfigException(
- "No Zookeeper quorum provided in the" + " configuration property "
- + RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
- }
- ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
-
- if (curatorClient == null) {
- curatorClient =
- CuratorFrameworkFactory.builder().connectString(registryQuorum)
- .sessionTimeoutMs(10000).retryPolicy(new RetryNTimes(5, 2000))
- .build();
- curatorClient.start();
- }
- return curatorClient;
- }
-
- private int actionHelp(String actionName, CommonArgs args)
- throws YarnException, IOException {
- throw new UsageException(CommonArgs.usage(args, actionName));
- }
-
- private void verifyNoLiveAppInRM(String appname, String action)
- throws IOException, YarnException {
- Set<String> types = new HashSet<>(1);
- types.add(YarnServiceConstants.APP_TYPE);
- Set<String> tags = null;
- if (appname != null) {
- tags = Collections.singleton(SliderUtils.createNameTag(appname));
- }
- GetApplicationsRequest request = GetApplicationsRequest.newInstance();
- request.setApplicationTypes(types);
- request.setApplicationTags(tags);
- request.setApplicationStates(liveStates);
- List<ApplicationReport> reports = yarnClient.getApplications(request);
- if (!reports.isEmpty()) {
- throw new YarnException(
- "Failed to " + action + " application, as " + appname
- + " already exists.");
- }
- }
-
- private ApplicationId submitApp(Application app)
- throws IOException, YarnException {
- String appName = app.getName();
- Configuration conf = getConfig();
- Path appRootDir = fs.buildClusterDirPath(app.getName());
-
- YarnClientApplication yarnApp = yarnClient.createApplication();
- ApplicationSubmissionContext submissionContext =
- yarnApp.getApplicationSubmissionContext();
- ServiceApiUtil.validateCompResourceSize(
- yarnApp.getNewApplicationResponse().getMaximumResourceCapability(),
- app);
-
- submissionContext.setKeepContainersAcrossApplicationAttempts(true);
- if (app.getLifetime() > 0) {
- Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>();
- appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime());
- submissionContext.setApplicationTimeouts(appTimeout);
- }
- submissionContext.setMaxAppAttempts(conf.getInt(
- YarnServiceConf.AM_RESTART_MAX, 2));
-
- Map<String, LocalResource> localResources = new HashMap<>();
-
- // copy local slideram-log4j.properties to hdfs and add to localResources
- boolean hasAMLog4j =
- addAMLog4jResource(appName, conf, localResources);
- // copy jars to hdfs and add to localResources
- addJarResource(appName, localResources);
- // add keytab if in secure env
- addKeytabResourceIfSecure(fs, localResources, conf, appName);
- if (LOG.isDebugEnabled()) {
- printLocalResources(localResources);
- }
- Map<String, String> env = addAMEnv(conf);
-
- // create AM CLI
- String cmdStr =
- buildCommandLine(appName, conf, appRootDir, hasAMLog4j);
- submissionContext.setResource(Resource.newInstance(YarnServiceConf
- .getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM,
- app.getConfiguration(), conf), 1));
- String queue = app.getQueue();
- if (StringUtils.isEmpty(queue)) {
- queue = conf.get(YARN_QUEUE, "default");
- }
- submissionContext.setQueue(queue);
- submissionContext.setApplicationName(appName);
- submissionContext.setApplicationType(YarnServiceConstants.APP_TYPE);
- Set<String> appTags =
- AbstractClientProvider.createApplicationTags(appName, null, null);
- if (!appTags.isEmpty()) {
- submissionContext.setApplicationTags(appTags);
- }
- ContainerLaunchContext amLaunchContext =
- Records.newRecord(ContainerLaunchContext.class);
- amLaunchContext.setCommands(Collections.singletonList(cmdStr));
- amLaunchContext.setEnvironment(env);
- amLaunchContext.setLocalResources(localResources);
- submissionContext.setAMContainerSpec(amLaunchContext);
- yarnClient.submitApplication(submissionContext);
- return submissionContext.getApplicationId();
- }
-
- private void printLocalResources(Map<String, LocalResource> map) {
- LOG.debug("Added LocalResource for localization: ");
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, LocalResource> entry : map.entrySet()) {
- builder.append(entry.getKey()).append(" -> ")
- .append(entry.getValue().getResource().getFile())
- .append(System.lineSeparator());
- }
- LOG.debug(builder.toString());
- }
-
- private String buildCommandLine(String appName, Configuration conf,
- Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException {
- JavaCommandLineBuilder CLI = new JavaCommandLineBuilder();
- CLI.forceIPv4().headless();
- //TODO CLI.setJVMHeap
- //TODO CLI.addJVMOPTS
- if (hasSliderAMLog4j) {
- CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME);
- CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- }
- CLI.add(ServiceMaster.class.getCanonicalName());
- CLI.add(ACTION_CREATE, appName);
- //TODO debugAM CLI.add(Arguments.ARG_DEBUG)
- CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, appName + ".json"));
- // pass the registry binding
- CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
- RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
- CLI.addMandatoryConfOption(conf, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
-
- // write out the path output
- CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
- String cmdStr = CLI.build();
- LOG.info("AM launch command: {}", cmdStr);
- return cmdStr;
- }
-
- private Map<String, String> addAMEnv(Configuration conf) throws IOException {
- Map<String, String> env = new HashMap<>();
- ClasspathConstructor classpath =
- buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
- .getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
- env.put("CLASSPATH", classpath.buildClasspath());
- env.put("LANG", "en_US.UTF-8");
- env.put("LC_ALL", "en_US.UTF-8");
- env.put("LANGUAGE", "en_US.UTF-8");
- String jaas = System.getenv("HADOOP_JAAS_DEBUG");
- if (jaas != null) {
- env.put("HADOOP_JAAS_DEBUG", jaas);
- }
- if (!UserGroupInformation.isSecurityEnabled()) {
- String userName = UserGroupInformation.getCurrentUser().getUserName();
- LOG.info("Run as user " + userName);
- // HADOOP_USER_NAME env is used by UserGroupInformation when log in
- // This env makes AM run as this user
- env.put("HADOOP_USER_NAME", userName);
- }
- LOG.info("AM env: \n{}", stringifyMap(env));
- return env;
- }
-
- protected Path addJarResource(String appName,
- Map<String, LocalResource> localResources)
- throws IOException, SliderException {
- Path libPath = fs.buildClusterDirPath(appName);
- ProviderUtils
- .addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
- libPath, "lib", false);
- Path dependencyLibTarGzip = fs.getDependencyTarGzip();
- if (fs.isFile(dependencyLibTarGzip)) {
- LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
- + dependencyLibTarGzip);
- SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
- } else {
- String[] libs = SliderUtils.getLibDirs();
- for (String libDirProp : libs) {
- ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
- libDirProp);
- }
- }
- return libPath;
- }
-
- private boolean addAMLog4jResource(String appName, Configuration conf,
- Map<String, LocalResource> localResources)
- throws IOException, BadClusterStateException {
- boolean hasAMLog4j = false;
- String hadoopConfDir =
- System.getenv(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
- if (hadoopConfDir != null) {
- File localFile =
- new File(hadoopConfDir, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
- if (localFile.exists()) {
- Path localFilePath = createLocalPath(localFile);
- Path appDirPath = fs.buildClusterDirPath(appName);
- Path remoteConfPath =
- new Path(appDirPath, YarnServiceConstants.SUBMITTED_CONF_DIR);
- Path remoteFilePath =
- new Path(remoteConfPath, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
- copy(conf, localFilePath, remoteFilePath);
- LocalResource localResource =
- fs.createAmResource(remoteConfPath, LocalResourceType.FILE);
- localResources.put(localFilePath.getName(), localResource);
- hasAMLog4j = true;
- } else {
- LOG.warn("AM log4j property file doesn't exist: " + localFile);
- }
- }
- return hasAMLog4j;
- }
-
- public int actionStart(String appName) throws YarnException, IOException {
- validateClusterName(appName);
- Path appDir = checkAppExistOnHdfs(appName);
- Application application = ServiceApiUtil.loadApplication(fs, appName);
- ServiceApiUtil.validateAndResolveApplication(application, fs, getConfig());
- // see if it is actually running and bail out;
- verifyNoLiveAppInRM(appName, "thaw");
- ApplicationId appId = submitApp(application);
- application.setId(appId.toString());
- // write app definition on to hdfs
- createDirAndPersistApp(appDir, application);
- return 0;
- }
-
- private Path checkAppNotExistOnHdfs(Application application)
- throws IOException, SliderException {
- Path appDir = fs.buildClusterDirPath(application.getName());
- fs.verifyDirectoryNonexistent(
- new Path(appDir, application.getName() + ".json"));
- return appDir;
- }
-
- private Path checkAppExistOnHdfs(String appName)
- throws IOException, SliderException {
- Path appDir = fs.buildClusterDirPath(appName);
- fs.verifyPathExists(new Path(appDir, appName + ".json"));
- return appDir;
- }
-
- private void createDirAndPersistApp(Path appDir, Application application)
- throws IOException, SliderException {
- FsPermission appDirPermission = new FsPermission("750");
- fs.createWithPermissions(appDir, appDirPermission);
- persistAppDef(appDir, application);
- }
-
- private void persistAppDef(Path appDir, Application application)
- throws IOException {
- Path appJson = new Path(appDir, application.getName() + ".json");
- ServiceApiUtil.jsonSerDeser
- .save(fs.getFileSystem(), appJson, application, true);
- LOG.info(
- "Persisted application " + application.getName() + " at " + appJson);
- }
-
- private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
- Map<String, LocalResource> localResource, Configuration conf,
- String appName) throws IOException, BadConfigException {
- if (!UserGroupInformation.isSecurityEnabled()) {
- return;
- }
- String keytabPreInstalledOnHost =
- conf.get(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
- if (StringUtils.isEmpty(keytabPreInstalledOnHost)) {
- String amKeytabName =
- conf.get(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
- String keytabDir = conf.get(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
- Path keytabPath =
- fileSystem.buildKeytabPath(keytabDir, amKeytabName, appName);
- if (fileSystem.getFileSystem().exists(keytabPath)) {
- LocalResource keytabRes =
- fileSystem.createAmResource(keytabPath, LocalResourceType.FILE);
- localResource
- .put(YarnServiceConstants.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
- LOG.info("Adding AM keytab on hdfs: " + keytabPath);
- } else {
- LOG.warn("No keytab file was found at {}.", keytabPath);
- if (conf.getBoolean(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
- throw new BadConfigException("No keytab file was found at %s.",
- keytabPath);
- } else {
- LOG.warn("The AM will be "
- + "started without a kerberos authenticated identity. "
- + "The application is therefore not guaranteed to remain "
- + "operational beyond 24 hours.");
- }
- }
- }
- }
-
- public String updateLifetime(String appName, long lifetime)
- throws YarnException, IOException {
- getAppId(appName);
- ApplicationId currentAppId = cachedAppIds.get(appName);
- ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
- if (report == null) {
- throw new YarnException("Application not found for " + appName);
- }
- ApplicationId appId = report.getApplicationId();
- LOG.info("Updating lifetime of an application: appName = " + appName
- + ", appId = " + appId + ", lifetime = " + lifetime);
- Map<ApplicationTimeoutType, String> map = new HashMap<>();
- String newTimeout =
- Times.formatISO8601(System.currentTimeMillis() + lifetime * 1000);
- map.put(ApplicationTimeoutType.LIFETIME, newTimeout);
- UpdateApplicationTimeoutsRequest request =
- UpdateApplicationTimeoutsRequest.newInstance(appId, map);
- yarnClient.updateApplicationTimeouts(request);
- LOG.info(
- "Successfully updated lifetime for an application: appName = " + appName
- + ", appId = " + appId + ". New expiry time in ISO8601 format is "
- + newTimeout);
- return newTimeout;
- }
-
- public Application getStatus(String appName)
- throws IOException, YarnException {
- validateClusterName(appName);
- ApplicationId currentAppId = getAppId(appName);
- ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId);
- ClientAMProtocol amProxy = getAMProxy(appName, appReport);
- Application appSpec;
- if (amProxy != null) {
- GetStatusResponseProto response =
- amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
- appSpec = ServiceApiUtil.jsonSerDeser.fromJson(response.getStatus());
- } else {
- appSpec = new Application();
- appSpec.setName(appName);
- }
- ApplicationTimeout lifetime =
- appReport.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME);
- if (lifetime != null) {
- appSpec.setLifetime(lifetime.getRemainingTime());
- }
- return appSpec;
- }
-
- public YarnClient getYarnClient() {
- return this.yarnClient;
- }
-
- public int actionDependency(ActionDependencyArgs args)
- throws IOException, YarnException {
- String currentUser = RegistryUtils.currentUser();
- LOG.info("Running command as user {}", currentUser);
-
- Path dependencyLibTarGzip = fs.getDependencyTarGzip();
-
- // Check if dependency has already been uploaded, in which case log
- // appropriately and exit success (unless overwrite has been requested)
- if (fs.isFile(dependencyLibTarGzip) && !args.overwrite) {
- System.out.println(String.format(
- "Dependency libs are already uploaded to %s. Use %s "
- + "if you want to re-upload", dependencyLibTarGzip.toUri(),
- Arguments.ARG_OVERWRITE));
- return EXIT_SUCCESS;
- }
-
- String[] libDirs = SliderUtils.getLibDirs();
- if (libDirs.length > 0) {
- File tempLibTarGzipFile = File.createTempFile(
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
- // copy all jars
- tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
-
- LOG.info("Version Info: " + VersionInfo.getBuildVersion());
- fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
- new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
- return EXIT_SUCCESS;
- } else {
- return EXIT_FALSE;
- }
- }
-
- // Get AMProxy with the appReport provided
- protected ClientAMProtocol getAMProxy(String appName, ApplicationReport report)
- throws IOException {
- if (!cachedAMProxies.containsKey(appName) && !StringUtils
- .isEmpty(report.getHost())) {
- insertAMProxy(appName, report.getHost(), report.getRpcPort());
- }
- return cachedAMProxies.get(appName);
- }
-
- // Get AMProxy without appReport provided - it'll getAppReport from RM
- protected ClientAMProtocol getAMProxy(String appName)
- throws IOException, YarnException {
- ApplicationId currentAppId = getAppId(appName);
-
- if (cachedAMProxies.containsKey(appName)) {
- return cachedAMProxies.get(appName);
- } else {
- ApplicationReport appReport =
- yarnClient.getApplicationReport(currentAppId);
- String host = appReport.getHost();
- int port = appReport.getRpcPort();
- if (!StringUtils.isEmpty(host)) {
- return insertAMProxy(appName, host, port);
- }
- return null;
- }
- }
-
- private ClientAMProtocol insertAMProxy(String appName, String host, int port)
- throws IOException {
- InetSocketAddress address =
- NetUtils.createSocketAddrForHost(host, port);
- ClientAMProtocol amProxy =
- ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
- UserGroupInformation.getCurrentUser(), rpc, address);
- cachedAMProxies.put(appName, amProxy);
- return amProxy;
- }
-
- private synchronized ApplicationId getAppId(String appName)
- throws IOException, YarnException {
- if (cachedAppIds.containsKey(appName)) {
- return cachedAppIds.get(appName);
- }
- Application persistedApp = ServiceApiUtil.loadApplication(fs, appName);
- if (persistedApp == null) {
- throw new YarnException("Application " + appName
- + " doesn't exist on hdfs. Please check if the app exists in RM");
- }
- ApplicationId currentAppId = ApplicationId.fromString(persistedApp.getId());
- cachedAppIds.put(appName, currentAppId);
- return currentAppId;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
deleted file mode 100644
index ea3bb0a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Base args for all actions
- */
-public abstract class AbstractActionArgs extends ArgOps implements Arguments {
- protected static final Logger log =
- LoggerFactory.getLogger(AbstractActionArgs.class);
-
-
- protected AbstractActionArgs() {
- }
-
- /**
- * URI/binding to the filesystem
- */
- @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
- description = "Filesystem Binding")
- public String filesystemBinding;
-
- @Parameter(names = {ARG_BASE_PATH},
- description = "Service base path on the filesystem",
- converter = PathArgumentConverter.class)
- public Path basePath;
-
- /**
- * This is the default parameter
- */
- @Parameter
- public final List<String> parameters = new ArrayList<>();
-
- /**
- * get the name: relies on arg 1 being the cluster name in all operations
- * @return the name argument, null if there is none
- */
- public String getClusterName() {
- return (parameters.isEmpty()) ? null : parameters.get(0);
- }
-
- /**
- -D name=value
-
- Define an configuration option which overrides any options in
- the configuration XML files of the image or in the image configuration
- directory. The values will be persisted.
- Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
-
- */
-
- @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
- public final List<String> definitions = new ArrayList<>();
-
- /**
- * System properties
- */
- @Parameter(names = {ARG_SYSPROP}, arity = 1,
- description = "system properties in the form name value" +
- " These are set after the JVM is started.")
- public final List<String> sysprops = new ArrayList<>(0);
-
-
- @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
- description = "Binding (usually hostname:port) of the YARN resource manager")
- public String manager;
-
-
- @Parameter(names = ARG_DEBUG, description = "Debug mode")
- public boolean debug = false;
-
- @Parameter(names = {ARG_HELP}, description = "Help", help = true)
- public boolean help = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 1;
- }
-
- /**
- * Get the name of the action
- * @return the action name
- */
- public abstract String getActionName() ;
-
- /**
- * Get the max #of params expected
- * @return the number of params in the {@link #parameters} field;
- */
- public int getMaxParams() {
- return getMinParams();
- }
-
- public void validate() throws BadCommandArgumentsException, UsageException {
-
- int minArgs = getMinParams();
- int actionArgSize = parameters.size();
- if (minArgs > actionArgSize) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
- ", Expected minimum " + minArgs + " but got " + actionArgSize);
- }
- int maxArgs = getMaxParams();
- if (maxArgs == -1) {
- maxArgs = minArgs;
- }
- if (actionArgSize > maxArgs) {
- String message = String.format("%s for action %s: limit is %d but saw %d: ",
- ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
- getActionName(), maxArgs,
- actionArgSize);
-
- log.error(message);
- int index = 1;
- StringBuilder buf = new StringBuilder(message);
- for (String actionArg : parameters) {
- log.error("[{}] \"{}\"", index++, actionArg);
- buf.append(" \"").append(actionArg).append("\" ");
- }
- throw new BadCommandArgumentsException(buf.toString());
- }
- }
-
- @Override
- public String toString() {
- return super.toString() + ": " + getActionName();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
deleted file mode 100644
index 457e357..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.yarn.service.client.params.ArgOps;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-
-/**
- * Base class for all the delegates
- */
-public class AbstractArgsDelegate extends ArgOps implements Arguments {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
deleted file mode 100644
index 017286f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.io.File;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Abstract Action to build things; shares args across build and
- * list
- */
-public abstract class AbstractClusterBuildingActionArgs
- extends AbstractActionArgs {
- @Parameter(names = {ARG_APPDEF},
- description = "Template application definition file in JSON format.")
- public File appDef;
-
- public File getAppDef() {
- return appDef;
- }
-
- @Parameter(names = {
- ARG_QUEUE }, description = "Queue to submit the application")
- public String queue;
-
- @Parameter(names = {
- ARG_LIFETIME }, description = "Lifetime of the application from the time of request")
- public long lifetime;
-
- @ParametersDelegate
- public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
-
- @ParametersDelegate
- public OptionArgsDelegate optionsDelegate =
- new OptionArgsDelegate();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
deleted file mode 100644
index c2ff545..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_BUILD},
- commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
-
-public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_BUILD;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
deleted file mode 100644
index 0097b4e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_CLIENT},
- commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
-
-public class ActionClientArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CLIENT;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install client")
- public boolean install;
-
- @Parameter(names = {ARG_NAME},
- description = "The name of the application")
- public String name;
-
- @Parameter(names = {ARG_PACKAGE},
- description = "Path to app package")
- public String packageURI;
-
- @Parameter(names = {ARG_DEST},
- description = "The location where to install the client")
- public File installLocation;
-
- @Parameter(names = {ARG_CONFIG},
- description = "Client configuration")
- public File clientConfig;
-
- /**
- * Get the min #of params expected
- *
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
deleted file mode 100644
index eecffb6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
deleted file mode 100644
index 51e07c9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY },
- commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY)
-public class ActionDependencyArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_DEPENDENCY;
- }
-
- @Parameter(names = { ARG_UPLOAD },
- description = "Upload AM and agent libraries to HDFS for this client")
- public boolean upload;
-
- @Parameter(names = { ARG_OVERWRITE },
- description = "Overwrite current uploaded dependency libs")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- *
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
-
- if (!upload) {
- throw new UsageException("Option " + ARG_UPLOAD + " is mandatory");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
deleted file mode 100644
index 8c41c04..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_DESTROY},
- commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
-
-public class ActionDestroyArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_DESTROY;
- }
-
- @Parameter(names = {ARG_FORCE},
- description = "force the operation")
- public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
deleted file mode 100644
index ba3c5a9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_EXISTS},
- commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
-
-public class ActionExistsArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_EXISTS;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "verify that the application is running")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "verify that the application is in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any application report")
- public File out;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
deleted file mode 100644
index b7acf58..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.List;
-import java.util.Map;
-
-@Parameters(commandNames = { SliderActions.ACTION_FLEX},
- commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
-
-public class ActionFlexArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_FLEX;
- }
-
- @ParametersDelegate
- public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
-
- /**
- * Get the component mapping (may be empty, but never null)
- * @return mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getComponentMap() throws
- BadCommandArgumentsException {
- return componentDelegate.getComponentMap();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
deleted file mode 100644
index aecf0eb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-@Parameters(commandNames = { SliderActions.ACTION_STOP },
- commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
-
-public class ActionFreezeArgs extends AbstractActionArgs implements
- WaitTimeAccessor {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STOP;
- }
-
- public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
- @ParametersDelegate
- public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
-
- @Override
- public int getWaittime() {
- return waitDelegate.getWaittime();
- }
-
- @Override
- public void setWaittime(int waittime) {
- waitDelegate.setWaittime(waittime);
- }
-
- @Parameter(names={ARG_MESSAGE},
- description = "reason for the operation")
- public String message = FREEZE_COMMAND_ISSUED;
-
- @Parameter(names = {ARG_FORCE},
- description = "force the operation")
- public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
deleted file mode 100644
index 51aa88a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-/**
- * The Help command
- */
-@Parameters(commandNames = { SliderActions.ACTION_HELP},
- commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
-public class ActionHelpArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_HELP;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
deleted file mode 100644
index 061121e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-@Parameters(commandNames = { SliderActions.ACTION_KDIAG},
- commandDescription = SliderActions.DESCRIBE_ACTION_KDIAG)
-
-public class ActionKDiagArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_KDIAG;
- }
-
- @Parameter(names = {ARG_SERVICES}, variableArity = true,
- description =" list of services to check")
- public List<String> services = new ArrayList<>();
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for report")
- public File out;
-
- @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
- public File keytab;
-
- @Parameter(names = {ARG_KEYLEN}, description = "minimum key length")
- public int keylen = 256;
-
- @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
- public String principal;
-
- @Parameter(names = {ARG_SECURE}, description = "Is security required")
- public boolean secure = false;
-
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (keytab != null && SliderUtils.isUnset(principal)) {
- throw new UsageException("Missing argument " + ARG_PRINCIPAL);
- }
- if (keytab == null && SliderUtils.isSet(principal)) {
- throw new UsageException("Missing argument " + ARG_KEYTAB);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
deleted file mode 100644
index 7e51457..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_KEYTAB},
- commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
-
-public class ActionKeytabArgs extends AbstractActionArgs {
-
- public ActionKeytabArgs() {
- super();
- }
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_INSTALL_KEYTAB;
- }
-
- @Parameter(names = {ARG_KEYTABINSTALL},
- description = "Install the keytab")
- public boolean install;
-
- @Parameter(names = {ARG_KEYTABDELETE},
- description = "Delete the keytab")
- public boolean delete;
-
- @Parameter(names = {ARG_KEYTABLIST},
- description = "List of installed keytabs")
- public boolean list;
-
- @Parameter(names = {ARG_KEYTAB},
- description = "Path or name of the keytab")
- public String keytab;
-
- @Parameter(names = {ARG_FOLDER},
- description = "The name of the folder in which to store the keytab")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
deleted file mode 100644
index 005c172..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_LIST},
- commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
-public class ActionListArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_LIST;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "List only live application instances")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "list only applications in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_VERBOSE},
- description = "print out information in details")
- public boolean verbose = false;
-
- @Parameter(names = {ARG_CONTAINERS},
- description = "List containers of an application instance")
- public boolean containers;
-
- @Parameter(names = {ARG_VERSION},
- description = "Filter containers by app version (used with " +
- ARG_CONTAINERS + ")")
- public String version;
-
- @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
- description = "Filter containers by component names (used with " +
- ARG_CONTAINERS + ")")
- public Set<String> components = new HashSet<>(0);
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/58] [abbrv] hadoop git commit: YARN-7191. Improve yarn-service
documentation. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7191. Improve yarn-service documentation. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88512097
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88512097
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88512097
Branch: refs/heads/yarn-native-services
Commit: 8851209788a4e110e899dd28f628657a21981db5
Parents: c70e5df
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Sep 27 15:08:33 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:17 2017 -0800
----------------------------------------------------------------------
.../src/site/markdown/yarn-service/Concepts.md | 47 +---
.../src/site/markdown/yarn-service/Overview.md | 3 +-
.../site/markdown/yarn-service/QuickStart.md | 34 +--
.../site/markdown/yarn-service/RegistryDNS.md | 166 +++++++++++++
.../markdown/yarn-service/ServiceDiscovery.md | 235 ++++++++-----------
5 files changed, 286 insertions(+), 199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88512097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
index 7b62c36..e567d03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
@@ -22,6 +22,8 @@ It also does all the heavy lifting work such as resolving the service definition
failed containers, monitoring components' healthiness and readiness, ensuring dependency start order across components, flexing up/down components,
upgrading components etc. The end goal of the framework is to make sure the service is up and running as the state that user desired.
+In addition, it leverages a lot of features in YARN core to accomplish scheduling constraints, such as
+affinity and anti-affinity scheduling, log aggregation for services, automatically restart a container if it fails, and do in-place upgrade of a container.
### A Restful API-Server for deploying/managing services on YARN
A restful API server is developed to allow users to deploy/manage their services on YARN via a simple JSON spec. This avoids users
@@ -34,44 +36,11 @@ support HA, distribute the load etc.
### Service Discovery
A DNS server is implemented to enable discovering services on YARN via the standard mechanism: DNS lookup.
-The DNS server essentially exposes the information in YARN service registry by translating them into DNS records such as A record and SRV record.
-Clients can discover the IPs of containers via standard DNS lookup.
-The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult
-to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [DNS doc](ServiceDiscovery.md)
-for more details.
-
-### Scheduling
-
-A host of scheduling features are being developed to support long running services.
-
-* Affinity and anti-affinity scheduling across containers ([YARN-6592](https://issues.apache.org/jira/browse/YARN-6592)).
-* Container resizing ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197))
-* Special handling of container preemption/reservation for services
-
-### Container auto-restarts
-
-[YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails.
-The service REST API provides users a way to enable NodeManager to automatically restart the container if it fails.
-The advantage is that it avoids the entire cycle of releasing the failed containers, re-asking new containers, re-do resource localizations and so on, which
-greatly minimizes container downtime.
+The framework posts container information such as hostname and ip into the [YARN service registry](../registry/index.md). And the DNS server essentially exposes the
+information in YARN service registry by translating them into DNS records such as A record and SRV record.
+Clients can then discover the IPs of containers via standard DNS lookup.
-### Container in-place upgrade
-
-[YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) aims to support upgrading containers in-place, that is, without losing the container allocations.
-It opens up a few APIs in NodeManager to allow ApplicationMasters to upgrade their containers via a simple API call.
-Under the hood, NodeManager does below steps:
-* Downloading the new resources such as jars, docker container images, new configurations.
-* Stop the old container.
-* Start the new container with the newly downloaded resources.
-
-At the time of writing this document, core changes are done but the feature is not usable end-to-end.
-
-### Resource Profiles
-
-In [YARN-3926](https://issues.apache.org/jira/browse/YARN-3926), YARN introduces Resource Profiles which extends the YARN resource model for easier
-resource-type management and profiles.
-It primarily solves two problems:
-* Make it easy to support new resource types such as network bandwith([YARN-2140](https://issues.apache.org/jira/browse/YARN-2140)), disks([YARN-2139](https://issues.apache.org/jira/browse/YARN-2139)).
- Under the hood, it unifies the scheduler codebase to essentially parameterize the resource types.
-* User can specify the container resource requirement by a profile name, rather than fiddling with varying resource-requirements for each resource type.
+The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult
+to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [Service Discovery doc](ServiceDiscovery.md)
+for more details.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88512097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
index 407fbc0..58daee5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -52,7 +52,8 @@ The benefits of combining these workloads are two-fold:
* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN.
* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN.
-* [Service Discovery](ServiceDiscovery.md): Deep dives into the YARN DNS internals.
+* [Service Discovery](ServiceDiscovery.md): Descirbes the service discovery mechanism on YARN.
+* [Registry DNS](RegistryDNS.md): Deep dives into the Registry DNS internals.
* [Examples](Examples.md): List some example service definitions (`Yarnfile`).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88512097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index ab415de..15df0cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -194,32 +194,10 @@ If you are building from source code, make sure you use `-Pyarn-ui` in the `mvn`
</property>
```
-## Service Discovery with YARN DNS
-YARN Service framework comes with a DNS server (backed by YARN Service Registry) which enables DNS based discovery of services deployed on YARN.
-That is, user can simply access their services in a well-defined naming format as below:
+# Try with Docker
+The above example is only for a non-docker container based service. YARN Service Framework also provides first-class support for managing docker based services.
+Most of the steps for managing docker based services are the same except that in docker the `Artifact` type for a component is `DOCKER` and the Artifact `id` is the name of the docker image.
+For details in how to setup docker on YARN, please check [Docker on YARN](../DockerContainers.md).
-```
-${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}
-```
-For example, in a cluster whose domain name is `yarncluster` (as defined by the `hadoop.registry.dns.domain-name` in `yarn-site.xml`), a service named `hbase` deployed by user `dev`
-with two components `hbasemaster` and `regionserver` can be accessed as below:
-
-This URL points to the usual hbase master UI
-```
-http://hbasemaster-0.hbase.dev.yarncluster:16010/master-status
-```
-
-
-Note that YARN service framework assigns COMPONENT_INSTANCE_NAME for each container in a sequence of monotonically increasing integers. For example, `hbasemaster-0` gets
-assigned `0` since it is the first and only instance for the `hbasemaster` component. In case of `regionserver` component, it can have multiple containers
- and so be named as such: `regionserver-0`, `regionserver-1`, `regionserver-2` ... etc
-
-`Disclaimer`: The DNS implementation is still experimental. It should not be used as a fully-functional corporate DNS.
-
-### Start the DNS server
-By default, the DNS runs on non-privileged port `5353`.
-If it is configured to use the standard privileged port `53`, the DNS server needs to be run as root:
-```
-sudo su - -c "yarn org.apache.hadoop.registry.server.dns.RegistryDNSServer > /${HADOOP_LOG_FOLDER}/registryDNS.log 2>&1 &" root
-```
-Please refer to [YARN DNS doc](ServicesDiscovery.md) for the full list of configurations.
\ No newline at end of file
+With docker support, it also opens up a set of new possibilities to implement features such as discovering service containers on YARN with DNS.
+Check [ServiceDiscovery](ServiceDiscovery.md) for more details.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88512097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
new file mode 100644
index 0000000..ef395fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
@@ -0,0 +1,166 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Registry DNS Server
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+## Introduction
+
+The Registry DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
+
+1. **Exposing existing service-discovery information via DNS** - Information provided in
+the current YARN service registry’s records will be converted into DNS entries, thus
+allowing users to discover information about YARN applications using standard DNS
+client mechanisms (e.g. a DNS SRV Record specifying the hostname and port
+number for services).
+2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
+standard DNS lookups. Given the availability of the records via DNS, container
+name-based communication will be facilitated (e.g. `curl
+http://solr-0.solr-service.devuser.yarncluster:8983/solr/admin/collections?action=LIST`).
+
+## Service Properties
+
+The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
+
+The following core functions are supported by the DNS-Server:
+
+### Functional properties
+
+1. Supports creation of DNS records for end-points of the deployed YARN applications
+2. Record names remain unchanged during restart of containers and/or applications
+3. Supports reverse lookups (name based on IP). Note, this works only for
+Docker containers because other containers share the IP of the host
+4. Supports security using the standards defined by The Domain Name System Security
+Extensions (DNSSEC)
+5. Highly available
+6. Scalable - The service provides the responsiveness (e.g. low-latency) required to
+respond to DNS queries (timeouts yield attempts to invoke other configured name
+servers).
+
+### Deployment properties
+
+1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as
+a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a
+primary DNS server and does not forward requests to other servers. Rather, a
+primary DNS server can be configured to forward a zone to the registry DNS
+server.
+2. The DNS Server exposes a port that can receive both TCP and UDP requests per
+DNS standards. The default port for DNS protocols is not in the restricted
+range (5353). However, existing DNS assets may only allow zone forwarding to
+non-custom ports. To support this, the registry DNS server can be started in
+privileged mode.
+
+## DNS Record Name Structure
+
+The DNS names of generated records are composed from the following elements
+(labels). Note that these elements must be compatible with DNS conventions
+(see “Preferred Name Syntax” in [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt)):
+
+* **domain** - the name of the cluster DNS domain. This name is provided as a
+configuration property. In addition, it is this name that is configured at a parent DNS
+server as the zone name for the defined registry DNS zone (the zone for which
+the parent DNS server will forward requests to registry DNS). E.g. yarncluster.com
+* **username** - the name of the application deployer. This name is the simple short-name (for
+e.g. the primary component of the Kerberos principal) associated with the user launching
+the application. As the username is one of the elements of DNS names, it is expected
+that this also conforms to DNS name conventions (RFC 1035 linked above), so it
+is converted to a valid DNS hostname entries using the punycode convention used
+for internationalized DNS.
+* **application name** - the name of the deployed YARN application. This name is inferred
+from the YARN registry path to the application's node. Application name,
+rather than application id, was chosen as a way of making it easy for users to refer to human-readable DNS
+names. This obviously mandates certain uniqueness properties on application names.
+* **container id** - the YARN assigned ID to a container (e.g.
+container_e3741_1454001598828_01_000004)
+* **component name** - the name assigned to the deployed component (for e.g. a master
+component). A component is a distributed element of an application or service that is
+launched in a YARN container (e.g. an HBase master). One can imagine multiple
+components within an application. A component name is not yet a first class concept in
+YARN, but is a very useful one that we are introducing here for the sake of registry DNS
+entries. Many frameworks like MapReduce, Slider already have component names
+(though, as mentioned, they are not yet supported in YARN in a first class fashion).
+* **api** - the api designation for the exposed endpoint
+
+### Notes about DNS Names
+
+* In most instances, the DNS names can be easily distinguished by the number of
+elements/labels that compose the name. The cluster’s domain name is always the last
+element. After that element is parsed out, reading from right to left, the first element
+maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
+“container” or suffix such as “api”. For example, an endpoint published as a
+management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
+* Unique application name (per user) is not currently supported/guaranteed by YARN, but
+it is supported by frameworks such as Apache Slider. The registry DNS service currently
+leverages the last element of the ZK path entry for the application as an
+application name. These application names have to be unique for a given user.
+
+## DNS Server Functionality
+
+The primary functions of the DNS service are illustrated in the following diagram:
+
+
+
+### DNS record creation
+The following figure illustrates at slightly greater detail the DNS record creation and registration sequence (NOTE: service record updates would follow a similar sequence of steps,
+distinguished only by the different event type):
+
+
+
+### DNS record removal
+Similarly, record removal follows a similar sequence
+
+
+
+(NOTE: The DNS Zone requires a record as an argument for the deletion method, thus
+requiring similar parsing logic to identify the specific records that should be removed).
+
+### DNS Service initialization
+* The DNS service initializes both UDP and TCP listeners on a configured port.
+If a port in the restricted range is desired (such as the standard DNS port
+53), the DNS service can be launched using jsvc as described in the section
+on starting the DNS server.
+* Subsequently, the DNS service listens for inbound DNS requests. Those requests are
+standard DNS requests from users or other DNS servers (for example, DNS servers that have the
+RegistryDNS service configured as a forwarder).
+
+## Start the DNS Server
+By default, the DNS server runs on non-privileged port `5353`. Start the server
+with:
+```
+yarn --daemon start registrydns
+```
+
+If the DNS server is configured to use the standard privileged port `53`, the
+environment variables YARN\_REGISTRYDNS\_SECURE\_USER and
+YARN\_REGISTRYDNS\_SECURE\_EXTRA\_OPTS must be uncommented in the yarn-env.sh
+file. The DNS server should then be launched as root and jsvc will be used to
+reduce the privileges of the daemon after the port has been bound.
+
+## Configuration
+The Registry DNS server reads its configuration properties from the yarn-site.xml file. The following are the DNS associated configuration properties:
+
+| Name | Description |
+| ------------ | ------------- |
+| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
+| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
+| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
+| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 5353. |
+| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
+| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
+| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
+| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
+| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
+| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
+| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88512097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
index 6318a07..a5dd0d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
@@ -12,139 +12,112 @@
limitations under the License. See accompanying LICENSE file.
-->
-# YARN DNS Server
-
-<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-
-## Introduction
-
-The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
-
-1. **Exposing existing service-discovery information via DNS** - Information provided in
-the current YARN service registry’s records will be converted into DNS entries, thus
-allowing users to discover information about YARN applications using standard DNS
-client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
-number for services).
-2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via
-standard DNS lookups. Given the availability of the records via DNS, container
-name-based communication will be facilitated (e.g. ‘curl
-http://myContainer.myDomain.com/endpoint’).
-
-## Service Properties
-
-The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
-
-The following core functions are supported by the DNS-Server:
-
-### Functional properties
-
-1. Supports creation of DNS records for end-points of the deployed YARN applications
-2. Record names remain unchanged during restart of containers and/or applications
-3. Supports reverse lookups (name based on IP). Note, this works only for Docker containers.
-4. Supports security using the standards defined by The Domain Name System Security
-Extensions (DNSSEC)
-5. Highly available
-6. Scalable - The service provides the responsiveness (e.g. low-latency) required to
-respond to DNS queries (timeouts yield attempts to invoke other configured name
-servers).
-
-### Deployment properties
-
-1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as
-a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a
-primary DNS server and does not forward requests to other servers.
-2. The DNS Server exposes a port that can receive both TCP and UDP requests per
-DNS standards. The default port for DNS protocols is in a restricted, administrative port
-range (5353), so the port is configurable for deployments in which the service may
-not be managed via an administrative account.
-
-## DNS Record Name Structure
-
-The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see “Preferred Name Syntax” in RFC 1035):
-
-* **domain** - the name of the cluster DNS domain. This name is provided as a
-configuration property. In addition, it is this name that is configured at a parent DNS
-server as the zone name for the defined yDNS zone (the zone for which the parent DNS
-server will forward requests to yDNS). E.g. yarncluster.com
-* **username** - the name of the application deployer. This name is the simple short-name (for
-e.g. the primary component of the Kerberos principal) associated with the user launching
-the application. As the username is one of the elements of DNS names, it is expected
-that this also confirms DNS name conventions (RFC 1035 linked above), so special translation is performed for names with special characters like hyphens and spaces.
-* **application name** - the name of the deployed YARN application. This name is inferred
-from the YARN registry path to the application's node. Application name, rather thn application id, was chosen as a way of making it easy for users to refer to human-readable DNS
-names. This obviously mandates certain uniqueness properties on application names.
-* **container id** - the YARN assigned ID to a container (e.g.
-container_e3741_1454001598828_01_000004)
-* **component name** - the name assigned to the deployed component (for e.g. a master
-component). A component is a distributed element of an application or service that is
-launched in a YARN container (e.g. an HBase master). One can imagine multiple
-components within an application. A component name is not yet a first class concept in
-YARN, but is a very useful one that we are introducing here for the sake of yDNS
-entries. Many frameworks like MapReduce, Slider already have component names
-(though, as mentioned, they are not yet supported in YARN in a first class fashion).
-* **api** - the api designation for the exposed endpoint
-
-### Notes about DNS Names
-
-* In most instances, the DNS names can be easily distinguished by the number of
-elements/labels that compose the name. The cluster’s domain name is always the last
-element. After that element is parsed out, reading from right to left, the first element
-maps to the application user and so on. Wherever it is not easily distinguishable, naming conventions are used to disambiguate the name using a prefix such as
-“container” or suffix such as “api”. For example, an endpoint published as a
-management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
-* Unique application name (per user) is not currently supported/guaranteed by YARN, but
-it is supported by frameworks such as Apache Slider. The yDNS service currently
-leverages the last element of the ZK path entry for the application as an
-application name. These application names have to be unique for a given user.
-
-## DNS Server Functionality
-
-The primary functions of the DNS service are illustrated in the following diagram:
-
-
-
-### DNS record creation
-The following figure illustrates at slightly greater detail the DNS record creation and registration sequence (NOTE: service record updates would follow a similar sequence of steps,
-distinguished only by the different event type):
-
-
-
-### DNS record removal
-Similarly, record removal follows a similar sequence
-
-
-
-(NOTE: The DNS Zone requires a record as an argument for the deletion method, thus
-requiring similar parsing logic to identify the specific records that should be removed).
-
-### DNS Service initialization
-* The DNS service initializes both UDP and TCP listeners on a configured port. As
-noted above, the default port of 5353 is in a restricted range that is only accessible to an
-account with administrative privileges.
-* Subsequently, the DNS service listens for inbound DNS requests. Those requests are
-standard DNS requests from users or other DNS servers (for example, DNS servers that have the
-YARN DNS service configured as a forwarder).
+# Service Discovery
+
+This document describes the mechanism of service discovery on YARN and the
+steps for enabling it.
+
+## Overview
+A [DNS server](RegistryDNS.md) is implemented to enable discovering services on YARN via
+the standard mechanism: DNS lookup.
+
+The framework ApplicationMaster posts the container information such as hostname and IP address into
+the YARN service registry. The DNS server exposes the information in YARN service registry by translating them into DNS
+records such as A record and SRV record. Clients can then discover the IPs of containers via standard DNS lookup.
+
+For non-docker containers (containers with null `Artifact` or with `Artifact` type set to `TARBALL`), since all containers on the same host share the same ip address,
+the DNS supports forward DNS lookup, but not support reverse DNS lookup.
+With docker, it supports both forward and reverse lookup, since each container
+can be configured to have its own unique IP. In addition, the DNS also supports configuring static zone files for both foward and reverse lookup.
+
+## Docker Container IP Management in Cluster
+To support the use-case of per container per IP, containers must be launched with `bridge` network. However, with `bridge` network, containers
+running on one node are not routable from other nodes by default. This is not an issue if you are only doing single node testing, however, for
+a multi-node environment, containers must be made routable from other nodes.
+
+There are several approaches to solve this depending on the platforms like GCE or AWS. Please refer to specific platform documentations for how to enable this.
+For on-prem cluster, one way to solve this issue is, on each node, configure the docker daemon to use a custom bridge say `br0` which is routable from all nodes.
+Also, assign an exclusive, contiguous range of IP addresses expressed in CIDR form e.g `172.21.195.240/26 (64 IPs)` to each docker
+daemon using the `fixed-cidr` option like below in the docker `daemon.json`:
+```
+"bridge": "br0"
+"fixed-cidr": "172.21.195.240/26"
+```
+Check how to [customize docker bridge network](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0/) for details.
+
+
+## Naming Convention with Registry DNS
+With the DNS support, user can simply access their services in a well-defined naming format as below:
+
+```
+${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}
+```
+For example, in a cluster whose domain name is `yarncluster` (as defined by the `hadoop.registry.dns.domain-name` in `yarn-site.xml`), a service named `hbase` deployed by user `devuser`
+with two components `hbasemaster` and `regionserver` can be accessed as below:
+
+This URL points to the usual hbase master UI
+```
+http://hbasemaster-0.hbase.devuser.yarncluster:16010/master-status
+```
+
+
+Note that YARN service framework assigns `COMPONENT_INSTANCE_NAME` for each container in a sequence of monotonically increasing integers. For example, `hbasemaster-0` gets
+assigned `0` since it is the first and only instance for the `hbasemaster` component. In case of `regionserver` component, it can have multiple containers
+ and so be named as such: `regionserver-0`, `regionserver-1`, `regionserver-2` ... etc
+
+`Disclaimer`: The DNS implementation is still experimental. It should not be used as a fully-functional DNS.
+
+
+## Configure Registry DNS
+
+Below is the set of configurations in `yarn-site.xml` required for enabling Registry DNS. A full list of properties can be found in the Configuration
+section of [Registry DNS](RegistryDNS.md).
+
+```
+ <property>
+ <description>The domain name for Hadoop cluster associated records.</description>
+ <name>hadoop.registry.dns.domain-name</name>
+ <value>ycluster</value>
+ </property>
+
+ <property>
+ <description>The port number for the DNS listener. The default port is 5353.
+ If the standard privileged port 53 is used, make sure start the DNS with jsvc support.</description>
+ <name>hadoop.registry.dns.bind-port</name>
+ <value>53</value>
+ </property>
+
+ <property>
+ <description>The DNS functionality is enabled for the cluster. Default is false.</description>
+ <name>hadoop.registry.dns.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>The network mask associated with the zone IP range. If specified, it is utilized to ascertain the
+ IP range possible and come up with an appropriate reverse zone name.</description>
+ <name>hadoop.registry.dns.zone-mask</name>
+ <value>255.255.255.0</value>
+ </property>
+
+ <property>
+ <description>An indicator of the IP range associated with the cluster containers. The setting is utilized for the
+ generation of the reverse zone name.</description>
+ <name>hadoop.registry.dns.zone-subnet</name>
+ <value>172.17.0</value>
+ </property>
+
+```
## Start the DNS Server
-By default, the DNS runs on non-privileged port `5353`.
-If it is configured to use the standard privileged port `53`, the DNS server needs to be run as root:
+By default, the DNS server runs on non-privileged port `5353`. Start the server
+with:
```
-sudo su - -c "yarn org.apache.hadoop.registry.server.dns.RegistryDNSServer > /${HADOOP_LOG_FOLDER}/registryDNS.log 2>&1 &" root
+yarn --daemon start registrydns
```
-## Configuration
-The YARN DNS server reads its configuration properties from the yarn-site.xml file. The following are the DNS associated configuration properties:
-
-| Name | Description |
-| ------------ | ------------- |
-| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
-| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
-| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
-| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 5353. However, since that port falls in a administrator-only range, typical deployments may need to specify an alternate port. |
-| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
-| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
-| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
-| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
-| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
-| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
-| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
+If the DNS server is configured to use the standard privileged port `53`, the
+environment variables `YARN_REGISTRYDNS_SECURE_USER` and
+`YARN_REGISTRYDNS_SECURE_EXTRA_OPTS` must be uncommented in the `yarn-env.sh`
+file. The DNS server should then be launched as `root` and jsvc will be used to
+reduce the privileges of the daemon after the port has been bound.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40ab068e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40ab068e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40ab068e
Branch: refs/heads/yarn-native-services
Commit: 40ab068eabe50875449d601471619a4e82fff86d
Parents: 2ba38ce
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Aug 28 09:59:55 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:13 2017 -0800
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 8 +-
.../assemblies/hadoop-yarn-services-dist.xml | 2 +-
hadoop-project/pom.xml | 2 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 14 +-
.../hadoop-yarn/conf/yarn-env.sh | 2 +-
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop/yarn/service/webapp/ApiServer.java | 274 +++
.../yarn/service/webapp/ApiServerWebApp.java | 159 ++
.../service/webapp/ApplicationApiService.java | 275 ---
.../service/webapp/ApplicationApiWebApp.java | 123 --
...RN-Simplified-V1-API-Layer-For-Services.yaml | 185 +-
.../src/main/scripts/run_rest_service.sh | 28 -
.../dev-support/findbugs-exclude.xml | 48 +
.../conf/yarnservice-log4j.properties | 62 +
.../hadoop-yarn-services-core/pom.xml | 408 +++++
.../hadoop/yarn/service/ClientAMProtocol.java | 40 +
.../hadoop/yarn/service/ClientAMService.java | 132 ++
.../yarn/service/ContainerFailureTracker.java | 89 +
.../hadoop/yarn/service/ServiceContext.java | 41 +
.../hadoop/yarn/service/ServiceMaster.java | 156 ++
.../hadoop/yarn/service/ServiceMetrics.java | 98 +
.../hadoop/yarn/service/ServiceScheduler.java | 654 +++++++
.../yarn/service/api/ServiceApiConstants.java | 69 +
.../yarn/service/api/records/Artifact.java | 160 ++
.../yarn/service/api/records/BaseResource.java | 52 +
.../yarn/service/api/records/Component.java | 412 +++++
.../yarn/service/api/records/ConfigFile.java | 225 +++
.../yarn/service/api/records/ConfigFormat.java | 67 +
.../yarn/service/api/records/Configuration.java | 225 +++
.../yarn/service/api/records/Container.java | 297 +++
.../service/api/records/ContainerState.java | 30 +
.../hadoop/yarn/service/api/records/Error.java | 129 ++
.../service/api/records/PlacementPolicy.java | 102 ++
.../service/api/records/ReadinessCheck.java | 175 ++
.../yarn/service/api/records/Resource.java | 159 ++
.../yarn/service/api/records/Service.java | 466 +++++
.../yarn/service/api/records/ServiceState.java | 33 +
.../yarn/service/api/records/ServiceStatus.java | 148 ++
.../yarn/service/client/ClientAMProxy.java | 57 +
.../hadoop/yarn/service/client/ServiceCLI.java | 112 ++
.../yarn/service/client/ServiceClient.java | 892 +++++++++
.../client/params/AbstractActionArgs.java | 158 ++
.../client/params/AbstractArgsDelegate.java | 28 +
.../AbstractClusterBuildingActionArgs.java | 58 +
.../service/client/params/ActionBuildArgs.java | 31 +
.../service/client/params/ActionClientArgs.java | 71 +
.../service/client/params/ActionCreateArgs.java | 33 +
.../client/params/ActionDependencyArgs.java | 65 +
.../client/params/ActionDestroyArgs.java | 37 +
.../service/client/params/ActionExistsArgs.java | 49 +
.../service/client/params/ActionFlexArgs.java | 50 +
.../service/client/params/ActionFreezeArgs.java | 56 +
.../service/client/params/ActionHelpArgs.java | 44 +
.../service/client/params/ActionKDiagArgs.java | 76 +
.../service/client/params/ActionKeytabArgs.java | 76 +
.../service/client/params/ActionListArgs.java | 76 +
.../client/params/ActionRegistryArgs.java | 218 +++
.../client/params/ActionResolveArgs.java | 153 ++
.../client/params/ActionResourceArgs.java | 70 +
.../service/client/params/ActionStatusArgs.java | 51 +
.../service/client/params/ActionThawArgs.java | 67 +
.../service/client/params/ActionTokensArgs.java | 78 +
.../service/client/params/ActionUpdateArgs.java | 32 +
.../yarn/service/client/params/ArgOps.java | 156 ++
.../yarn/service/client/params/Arguments.java | 103 ++
.../yarn/service/client/params/ClientArgs.java | 252 +++
.../yarn/service/client/params/CommonArgs.java | 282 +++
.../client/params/ComponentArgsDelegate.java | 52 +
.../client/params/DontSplitArguments.java | 34 +
.../client/params/LaunchArgsAccessor.java | 30 +
.../client/params/LaunchArgsDelegate.java | 51 +
.../client/params/OptionArgsDelegate.java | 66 +
.../client/params/PathArgumentConverter.java | 34 +
.../service/client/params/SliderAMArgs.java | 57 +
.../client/params/SliderAMCreateAction.java | 73 +
.../service/client/params/SliderActions.java | 82 +
.../service/client/params/WaitArgsDelegate.java | 42 +
.../service/client/params/WaitTimeAccessor.java | 24 +
.../yarn/service/component/Component.java | 494 +++++
.../yarn/service/component/ComponentEvent.java | 83 +
.../service/component/ComponentEventType.java | 26 +
.../yarn/service/component/ComponentState.java | 25 +
.../component/instance/ComponentInstance.java | 493 +++++
.../instance/ComponentInstanceEvent.java | 58 +
.../instance/ComponentInstanceEventType.java | 26 +
.../component/instance/ComponentInstanceId.java | 91 +
.../instance/ComponentInstanceState.java | 26 +
.../yarn/service/conf/RestApiConstants.java | 43 +
.../yarn/service/conf/SliderExitCodes.java | 88 +
.../yarn/service/conf/YarnServiceConf.java | 101 ++
.../yarn/service/conf/YarnServiceConstants.java | 90 +
.../containerlaunch/AbstractLauncher.java | 271 +++
.../containerlaunch/ClasspathConstructor.java | 172 ++
.../containerlaunch/CommandLineBuilder.java | 86 +
.../containerlaunch/ContainerLaunchService.java | 101 ++
.../containerlaunch/CredentialUtils.java | 319 ++++
.../containerlaunch/JavaCommandLineBuilder.java | 181 ++
.../exceptions/BadClusterStateException.java | 36 +
.../BadCommandArgumentsException.java | 30 +
.../service/exceptions/BadConfigException.java | 39 +
.../yarn/service/exceptions/ErrorStrings.java | 42 +
.../service/exceptions/ExitCodeProvider.java | 32 +
.../service/exceptions/LauncherExitCodes.java | 196 ++
.../exceptions/RestApiErrorMessages.java | 92 +
.../exceptions/ServiceLaunchException.java | 73 +
.../service/exceptions/SliderException.java | 66 +
.../yarn/service/exceptions/UsageException.java | 34 +
.../pb/client/ClientAMProtocolPBClientImpl.java | 91 +
.../impl/pb/service/ClientAMProtocolPB.java | 29 +
.../service/ClientAMProtocolPBServiceImpl.java | 70 +
.../yarn/service/monitor/ServiceMonitor.java | 147 ++
.../yarn/service/monitor/probe/HttpProbe.java | 110 ++
.../service/monitor/probe/LogEntryBuilder.java | 76 +
.../yarn/service/monitor/probe/MonitorKeys.java | 66 +
.../service/monitor/probe/MonitorUtils.java | 84 +
.../yarn/service/monitor/probe/PortProbe.java | 98 +
.../yarn/service/monitor/probe/Probe.java | 100 ++
.../yarn/service/monitor/probe/ProbeStatus.java | 160 ++
.../provider/AbstractClientProvider.java | 122 ++
.../provider/AbstractProviderService.java | 109 ++
.../yarn/service/provider/ProviderFactory.java | 76 +
.../yarn/service/provider/ProviderService.java | 39 +
.../yarn/service/provider/ProviderUtils.java | 402 +++++
.../defaultImpl/DefaultClientProvider.java | 46 +
.../defaultImpl/DefaultProviderFactory.java | 51 +
.../defaultImpl/DefaultProviderService.java | 36 +
.../provider/docker/DockerClientProvider.java | 53 +
.../service/provider/docker/DockerKeys.java | 30 +
.../provider/docker/DockerProviderFactory.java | 52 +
.../provider/docker/DockerProviderService.java | 57 +
.../provider/tarball/TarballClientProvider.java | 65 +
.../tarball/TarballProviderFactory.java | 52 +
.../tarball/TarballProviderService.java | 48 +
.../registry/CustomRegistryConstants.java | 57 +
.../registry/YarnRegistryViewForProviders.java | 225 +++
.../timelineservice/ServiceMetricsSink.java | 102 ++
.../ServiceTimelineEntityType.java | 39 +
.../timelineservice/ServiceTimelineEvent.java | 34 +
.../ServiceTimelineMetricsConstants.java | 92 +
.../ServiceTimelinePublisher.java | 368 ++++
.../service/timelineservice/package-info.java | 27 +
.../utils/ApplicationReportSerDeser.java | 56 +
.../service/utils/ClientRegistryBinder.java | 201 +++
.../hadoop/yarn/service/utils/Comparators.java | 62 +
.../hadoop/yarn/service/utils/ConfigHelper.java | 157 ++
.../hadoop/yarn/service/utils/ConfigUtils.java | 97 +
.../yarn/service/utils/CoreFileSystem.java | 521 ++++++
.../hadoop/yarn/service/utils/Duration.java | 109 ++
.../hadoop/yarn/service/utils/JsonSerDeser.java | 249 +++
.../yarn/service/utils/KerberosDiags.java | 680 +++++++
.../yarn/service/utils/PatternValidator.java | 58 +
.../hadoop/yarn/service/utils/PortScanner.java | 113 ++
.../service/utils/PublishedConfiguration.java | 196 ++
.../utils/PublishedConfigurationOutputter.java | 212 +++
.../utils/SerializedApplicationReport.java | 98 +
.../yarn/service/utils/ServiceApiUtil.java | 446 +++++
.../service/utils/ServiceRegistryUtils.java | 71 +
.../yarn/service/utils/SliderFileSystem.java | 51 +
.../hadoop/yarn/service/utils/SliderUtils.java | 1654 +++++++++++++++++
.../yarn/service/utils/ZookeeperUtils.java | 146 ++
.../src/main/proto/ClientAMProtocol.proto | 56 +
.../hadoop/yarn/service/MockServiceAM.java | 221 +++
.../hadoop/yarn/service/ServiceTestUtils.java | 59 +
.../hadoop/yarn/service/TestServiceApiUtil.java | 530 ++++++
.../yarn/service/TestYarnNativeServices.java | 472 +++++
.../client/TestBuildExternalComponents.java | 128 ++
.../yarn/service/client/TestServiceCLI.java | 155 ++
.../yarn/service/conf/ExampleAppJson.java | 65 +
.../yarn/service/conf/TestAppJsonResolve.java | 222 +++
.../service/conf/TestLoadExampleAppJson.java | 78 +
.../service/conf/TestValidateServiceNames.java | 126 ++
.../service/monitor/TestServiceMonitor.java | 104 ++
.../providers/TestAbstractClientProvider.java | 118 ++
.../service/providers/TestProviderFactory.java | 76 +
.../TestServiceTimelinePublisher.java | 293 +++
.../src/test/resources/example-app.json | 15 +
.../service/conf/examples/app-override.json | 72 +
.../hadoop/yarn/service/conf/examples/app.json | 48 +
.../yarn/service/conf/examples/default.json | 16 +
.../yarn/service/conf/examples/external0.json | 8 +
.../yarn/service/conf/examples/external1.json | 30 +
.../yarn/service/conf/examples/external2.json | 22 +
.../src/test/resources/yarn-site.xml | 19 +
.../hadoop-yarn-services/pom.xml | 38 +
.../dev-support/findbugs-exclude.xml | 48 -
.../conf/yarnservice-log4j.properties | 62 -
.../hadoop-yarn-slider-core/pom.xml | 409 -----
.../src/assembly/executable-jar.xml | 47 -
.../hadoop/yarn/service/ClientAMProtocol.java | 40 -
.../hadoop/yarn/service/ClientAMService.java | 132 --
.../yarn/service/ContainerFailureTracker.java | 89 -
.../hadoop/yarn/service/ServiceContext.java | 41 -
.../hadoop/yarn/service/ServiceMaster.java | 157 --
.../hadoop/yarn/service/ServiceScheduler.java | 655 -------
.../api/constants/ServiceApiConstants.java | 69 -
.../yarn/service/api/records/Application.java | 466 -----
.../service/api/records/ApplicationState.java | 33 -
.../service/api/records/ApplicationStatus.java | 148 --
.../yarn/service/api/records/Artifact.java | 160 --
.../yarn/service/api/records/BaseResource.java | 52 -
.../yarn/service/api/records/Component.java | 412 -----
.../yarn/service/api/records/ConfigFile.java | 225 ---
.../yarn/service/api/records/ConfigFormat.java | 67 -
.../yarn/service/api/records/Configuration.java | 225 ---
.../yarn/service/api/records/Container.java | 297 ---
.../service/api/records/ContainerState.java | 30 -
.../hadoop/yarn/service/api/records/Error.java | 129 --
.../service/api/records/PlacementPolicy.java | 102 --
.../service/api/records/ReadinessCheck.java | 175 --
.../yarn/service/api/records/Resource.java | 159 --
.../yarn/service/client/ClientAMProxy.java | 49 -
.../hadoop/yarn/service/client/ServiceCLI.java | 104 --
.../yarn/service/client/ServiceClient.java | 872 ---------
.../client/params/AbstractActionArgs.java | 158 --
.../client/params/AbstractArgsDelegate.java | 28 -
.../AbstractClusterBuildingActionArgs.java | 58 -
.../service/client/params/ActionBuildArgs.java | 31 -
.../service/client/params/ActionClientArgs.java | 71 -
.../service/client/params/ActionCreateArgs.java | 33 -
.../client/params/ActionDependencyArgs.java | 65 -
.../client/params/ActionDestroyArgs.java | 37 -
.../service/client/params/ActionExistsArgs.java | 49 -
.../service/client/params/ActionFlexArgs.java | 50 -
.../service/client/params/ActionFreezeArgs.java | 56 -
.../service/client/params/ActionHelpArgs.java | 44 -
.../service/client/params/ActionKDiagArgs.java | 76 -
.../service/client/params/ActionKeytabArgs.java | 76 -
.../service/client/params/ActionListArgs.java | 76 -
.../client/params/ActionRegistryArgs.java | 218 ---
.../client/params/ActionResolveArgs.java | 153 --
.../client/params/ActionResourceArgs.java | 70 -
.../service/client/params/ActionStatusArgs.java | 51 -
.../service/client/params/ActionThawArgs.java | 67 -
.../service/client/params/ActionTokensArgs.java | 78 -
.../service/client/params/ActionUpdateArgs.java | 32 -
.../yarn/service/client/params/ArgOps.java | 156 --
.../yarn/service/client/params/Arguments.java | 103 --
.../yarn/service/client/params/ClientArgs.java | 252 ---
.../yarn/service/client/params/CommonArgs.java | 282 ---
.../client/params/ComponentArgsDelegate.java | 52 -
.../client/params/DontSplitArguments.java | 34 -
.../client/params/LaunchArgsAccessor.java | 30 -
.../client/params/LaunchArgsDelegate.java | 51 -
.../client/params/OptionArgsDelegate.java | 66 -
.../client/params/PathArgumentConverter.java | 34 -
.../service/client/params/SliderAMArgs.java | 57 -
.../client/params/SliderAMCreateAction.java | 73 -
.../service/client/params/SliderActions.java | 82 -
.../service/client/params/WaitArgsDelegate.java | 42 -
.../service/client/params/WaitTimeAccessor.java | 24 -
.../service/compinstance/ComponentInstance.java | 493 -----
.../compinstance/ComponentInstanceEvent.java | 58 -
.../ComponentInstanceEventType.java | 27 -
.../compinstance/ComponentInstanceId.java | 91 -
.../compinstance/ComponentInstanceState.java | 26 -
.../yarn/service/component/Component.java | 493 -----
.../yarn/service/component/ComponentEvent.java | 83 -
.../service/component/ComponentEventType.java | 26 -
.../yarn/service/component/ComponentState.java | 25 -
.../yarn/service/conf/RestApiConstants.java | 43 -
.../yarn/service/conf/SliderExitCodes.java | 88 -
.../yarn/service/conf/YarnServiceConf.java | 97 -
.../yarn/service/conf/YarnServiceConstants.java | 90 -
.../containerlaunch/AbstractLauncher.java | 271 ---
.../containerlaunch/ClasspathConstructor.java | 172 --
.../containerlaunch/CommandLineBuilder.java | 86 -
.../containerlaunch/ContainerLaunchService.java | 101 --
.../containerlaunch/CredentialUtils.java | 319 ----
.../containerlaunch/JavaCommandLineBuilder.java | 181 --
.../exceptions/BadClusterStateException.java | 36 -
.../BadCommandArgumentsException.java | 30 -
.../service/exceptions/BadConfigException.java | 39 -
.../yarn/service/exceptions/ErrorStrings.java | 57 -
.../service/exceptions/ExitCodeProvider.java | 32 -
.../service/exceptions/LauncherExitCodes.java | 196 --
.../exceptions/RestApiErrorMessages.java | 92 -
.../exceptions/ServiceLaunchException.java | 73 -
.../service/exceptions/SliderException.java | 66 -
.../yarn/service/exceptions/UsageException.java | 34 -
.../pb/client/ClientAMProtocolPBClientImpl.java | 91 -
.../impl/pb/service/ClientAMProtocolPB.java | 29 -
.../service/ClientAMProtocolPBServiceImpl.java | 70 -
.../yarn/service/metrics/ServiceMetrics.java | 101 --
.../provider/AbstractClientProvider.java | 122 --
.../provider/AbstractProviderService.java | 109 --
.../yarn/service/provider/ProviderFactory.java | 76 -
.../yarn/service/provider/ProviderService.java | 39 -
.../yarn/service/provider/ProviderUtils.java | 402 -----
.../defaultImpl/DefaultClientProvider.java | 46 -
.../defaultImpl/DefaultProviderFactory.java | 51 -
.../defaultImpl/DefaultProviderService.java | 36 -
.../provider/docker/DockerClientProvider.java | 53 -
.../service/provider/docker/DockerKeys.java | 30 -
.../provider/docker/DockerProviderFactory.java | 52 -
.../provider/docker/DockerProviderService.java | 57 -
.../provider/tarball/TarballClientProvider.java | 65 -
.../tarball/TarballProviderFactory.java | 52 -
.../tarball/TarballProviderService.java | 48 -
.../registry/CustomRegistryConstants.java | 57 -
.../registry/YarnRegistryViewForProviders.java | 225 ---
.../yarn/service/rest/BaseRestClient.java | 149 --
.../yarn/service/rest/ExceptionConverter.java | 128 --
.../hadoop/yarn/service/rest/HttpVerb.java | 57 -
.../rest/SliderURLConnectionFactory.java | 176 --
.../yarn/service/rest/UgiJerseyBinding.java | 153 --
.../service/rest/UrlConnectionOperations.java | 83 -
.../service/servicemonitor/ServiceMonitor.java | 148 --
.../service/servicemonitor/probe/HttpProbe.java | 110 --
.../servicemonitor/probe/LogEntryBuilder.java | 76 -
.../servicemonitor/probe/MonitorKeys.java | 66 -
.../servicemonitor/probe/MonitorUtils.java | 84 -
.../service/servicemonitor/probe/PortProbe.java | 98 -
.../service/servicemonitor/probe/Probe.java | 100 --
.../servicemonitor/probe/ProbeStatus.java | 160 --
.../timelineservice/ServiceMetricsSink.java | 102 --
.../ServiceTimelineEntityType.java | 39 -
.../timelineservice/ServiceTimelineEvent.java | 34 -
.../ServiceTimelineMetricsConstants.java | 92 -
.../ServiceTimelinePublisher.java | 368 ----
.../service/timelineservice/package-info.java | 27 -
.../utils/ApplicationReportSerDeser.java | 56 -
.../service/utils/ClientRegistryBinder.java | 201 ---
.../hadoop/yarn/service/utils/Comparators.java | 62 -
.../hadoop/yarn/service/utils/ConfigHelper.java | 157 --
.../hadoop/yarn/service/utils/ConfigUtils.java | 97 -
.../yarn/service/utils/CoreFileSystem.java | 521 ------
.../hadoop/yarn/service/utils/Duration.java | 109 --
.../hadoop/yarn/service/utils/JsonSerDeser.java | 249 ---
.../yarn/service/utils/KerberosDiags.java | 680 -------
.../yarn/service/utils/PatternValidator.java | 58 -
.../hadoop/yarn/service/utils/PortScanner.java | 113 --
.../service/utils/PublishedConfiguration.java | 196 --
.../utils/PublishedConfigurationOutputter.java | 212 ---
.../utils/SerializedApplicationReport.java | 98 -
.../yarn/service/utils/ServiceApiUtil.java | 443 -----
.../service/utils/ServiceRegistryUtils.java | 71 -
.../yarn/service/utils/SliderFileSystem.java | 51 -
.../hadoop/yarn/service/utils/SliderUtils.java | 1699 ------------------
.../yarn/service/utils/ZookeeperUtils.java | 146 --
.../src/main/proto/ClientAMProtocol.proto | 56 -
.../hadoop/yarn/service/MockServiceAM.java | 221 ---
.../hadoop/yarn/service/ServiceTestUtils.java | 59 -
.../hadoop/yarn/service/TestServiceApiUtil.java | 529 ------
.../yarn/service/TestYarnNativeServices.java | 472 -----
.../client/TestBuildExternalComponents.java | 128 --
.../yarn/service/client/TestServiceCLI.java | 139 --
.../yarn/service/conf/ExampleAppJson.java | 65 -
.../yarn/service/conf/TestAppJsonResolve.java | 224 ---
.../service/conf/TestLoadExampleAppJson.java | 78 -
.../service/conf/TestValidateServiceNames.java | 123 --
.../providers/TestAbstractClientProvider.java | 118 --
.../service/providers/TestProviderFactory.java | 76 -
.../servicemonitor/TestServiceMonitor.java | 104 --
.../TestServiceTimelinePublisher.java | 293 ---
.../src/test/resources/example-app.json | 15 -
.../src/test/resources/log4j.properties | 66 -
.../service/conf/examples/app-override.json | 72 -
.../hadoop/yarn/service/conf/examples/app.json | 47 -
.../yarn/service/conf/examples/default.json | 16 -
.../yarn/service/conf/examples/external0.json | 8 -
.../yarn/service/conf/examples/external1.json | 30 -
.../yarn/service/conf/examples/external2.json | 22 -
.../yarn/service/provider/docker/appConfig.json | 42 -
.../yarn/service/provider/docker/resources.json | 16 -
.../yarn/service/provider/docker/test.template | 16 -
.../src/test/resources/yarn-site.xml | 19 -
.../hadoop-yarn-slider/pom.xml | 39 -
.../hadoop-yarn-applications/pom.xml | 2 +-
.../native-services/NativeServicesDiscovery.md | 14 +
.../native-services/NativeServicesIntro.md | 13 +
hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +-
371 files changed, 23798 insertions(+), 24702 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 4483fa8..8aeeabd 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -87,19 +87,19 @@
</includes>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target</directory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
<include>*-sources.jar</include>
</includes>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf</directory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf</directory>
<outputDirectory>etc/hadoop</outputDirectory>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/lib/slider</outputDirectory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version}</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/lib/services</outputDirectory>
</fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
index 5de45a9..1b81f98 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml
@@ -17,7 +17,7 @@
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>hadoop-yarn-slider-dist</id>
+ <id>hadoop-yarn-services-dist</id>
<formats>
<format>dir</format>
</formats>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index f5b550f..3923183 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -428,7 +428,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider-core</artifactId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 785f714..0a53455 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -49,7 +49,7 @@ function hadoop_usage
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "servicesapi" "run yarn-native-service rest server"
+ hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "service" "run a service"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
@@ -151,14 +151,14 @@ function yarncmd_case
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
- servicesapi)
+ apiserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*'
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApplicationApiWebApp'
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
@@ -171,11 +171,11 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
;;
service)
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*'
HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 02aec3b..90a87bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -159,4 +159,4 @@
#
# See ResourceManager for some examples
#
-#export YARN_SERVICESAPI_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-servicesapi.log-$(date +'%Y%m%d%H%M')"
+#export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index fb2f2ba..7d9f15c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -91,7 +91,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider-core</artifactId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
new file mode 100644
index 0000000..f55e3f1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.webapp;
+
+import com.google.inject.Singleton;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
+
+/**
+ * The rest API endpoints for users to manage services on YARN.
+ */
+@Singleton
+@Path(CONTEXT_ROOT)
+public class ApiServer {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApiServer.class);
+ private static Configuration YARN_CONFIG = new YarnConfiguration();
+ private static ServiceClient SERVICE_CLIENT;
+
+ static {
+ init();
+ }
+
+ // initialize all the common resources - order is important
+ private static void init() {
+ SERVICE_CLIENT = new ServiceClient();
+ SERVICE_CLIENT.init(YARN_CONFIG);
+ SERVICE_CLIENT.start();
+ }
+
+ @GET
+ @Path(VERSION)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+ public Response getVersion() {
+ String version = VersionInfo.getBuildVersion();
+ LOG.info(version);
+ return Response.ok(version).build();
+ }
+
+ @POST
+ @Path(SERVICE_ROOT_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response createService(Service service) {
+ LOG.info("POST: createService = {}", service);
+ ServiceStatus serviceStatus = new ServiceStatus();
+ try {
+ ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service);
+ LOG.info("Successfully created service " + service.getName()
+ + " applicationId = " + applicationId);
+ serviceStatus.setState(ServiceState.ACCEPTED);
+ serviceStatus.setUri(
+ CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
+ .getName());
+ return Response.status(Status.CREATED).entity(serviceStatus).build();
+ } catch (IllegalArgumentException e) {
+ serviceStatus.setDiagnostics(e.getMessage());
+ return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
+ .build();
+ } catch (Exception e) {
+ String message = "Failed to create service " + service.getName();
+ LOG.error(message, e);
+ serviceStatus.setDiagnostics(message + ": " + e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(serviceStatus).build();
+ }
+ }
+
+ @GET
+ @Path(SERVICE_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response getService(@PathParam(SERVICE_NAME) String appName) {
+ LOG.info("GET: getService for appName = {}", appName);
+ ServiceStatus serviceStatus = new ServiceStatus();
+ try {
+ Service app = SERVICE_CLIENT.getStatus(appName);
+ return Response.ok(app).build();
+ } catch (IllegalArgumentException e) {
+ serviceStatus.setDiagnostics(e.getMessage());
+ serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
+ return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+ .build();
+ } catch (Exception e) {
+ LOG.error("Get service failed", e);
+ serviceStatus
+ .setDiagnostics("Failed to retrieve service: " + e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(serviceStatus).build();
+ }
+ }
+
+ @DELETE
+ @Path(SERVICE_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response deleteService(@PathParam(SERVICE_NAME) String appName) {
+ LOG.info("DELETE: deleteService for appName = {}", appName);
+ return stopService(appName, true);
+ }
+
+ private Response stopService(String appName, boolean destroy) {
+ try {
+ SERVICE_CLIENT.actionStop(appName, destroy);
+ if (destroy) {
+ SERVICE_CLIENT.actionDestroy(appName);
+ LOG.info("Successfully deleted service {}", appName);
+ } else {
+ LOG.info("Successfully stopped service {}", appName);
+ }
+ return Response.status(Status.NO_CONTENT).build();
+ } catch (ApplicationNotFoundException e) {
+ ServiceStatus serviceStatus = new ServiceStatus();
+ serviceStatus.setDiagnostics(
+ "Service " + appName + " not found " + e.getMessage());
+ return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+ .build();
+ } catch (Exception e) {
+ ServiceStatus serviceStatus = new ServiceStatus();
+ serviceStatus.setDiagnostics(e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(serviceStatus).build();
+ }
+ }
+
+ @PUT
+ @Path(COMPONENT_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+ public Response updateComponent(@PathParam(SERVICE_NAME) String appName,
+ @PathParam(COMPONENT_NAME) String componentName, Component component) {
+
+ if (component.getNumberOfContainers() < 0) {
+ return Response.status(Status.BAD_REQUEST).entity(
+ "Service = " + appName + ", Component = " + component.getName()
+ + ": Invalid number of containers specified " + component
+ .getNumberOfContainers()).build();
+ }
+ try {
+ Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
+ Collections.singletonMap(component.getName(),
+ component.getNumberOfContainers()));
+ return Response.ok().entity(
+ "Updating component " + componentName + " size from " + original
+ .get(componentName) + " to " + component.getNumberOfContainers())
+ .build();
+ } catch (YarnException | IOException e) {
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics(e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
+ .build();
+ }
+ }
+
+ @PUT
+ @Path(SERVICE_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response updateService(@PathParam(SERVICE_NAME) String appName,
+ Service updateServiceData) {
+ LOG.info("PUT: updateService for app = {} with data = {}", appName,
+ updateServiceData);
+
+ // Ignore the app name provided in updateServiceData and always use appName
+ // path param
+ updateServiceData.setName(appName);
+
+ // For STOP the app should be running. If already stopped then this
+ // operation will be a no-op. For START it should be in stopped state.
+ // If already running then this operation will be a no-op.
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STOPPED) {
+ return stopService(appName, false);
+ }
+
+ // If a START is requested
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STARTED) {
+ return startService(appName);
+ }
+
+ // If new lifetime value specified then update it
+ if (updateServiceData.getLifetime() != null
+ && updateServiceData.getLifetime() > 0) {
+ return updateLifetime(appName, updateServiceData);
+ }
+
+ // flex a single component app
+ if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil
+ .hasComponent(updateServiceData)) {
+ Component defaultComp = ServiceApiUtil.createDefaultComponent(updateServiceData);
+ return updateComponent(updateServiceData.getName(), defaultComp.getName(),
+ defaultComp);
+ }
+
+ // If nothing happens consider it a no-op
+ return Response.status(Status.NO_CONTENT).build();
+ }
+
+ private Response updateLifetime(String appName, Service updateAppData) {
+ try {
+ String newLifeTime =
+ SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
+ return Response.ok("Service " + appName + " lifeTime is successfully updated to "
+ + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
+ } catch (Exception e) {
+ String message =
+ "Failed to update service (" + appName + ") lifetime ("
+ + updateAppData.getLifetime() + ")";
+ LOG.error(message, e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(message + " : " + e.getMessage()).build();
+ }
+ }
+
+ private Response startService(String appName) {
+ try {
+ SERVICE_CLIENT.actionStart(appName);
+ LOG.info("Successfully started service " + appName);
+ return Response.ok("Service " + appName + " is successfully started").build();
+ } catch (Exception e) {
+ String message = "Failed to start service " + appName;
+ LOG.info(message, e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(message + ": " + e.getMessage()).build();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
new file mode 100644
index 0000000..b226df7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.webapp;
+
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.eclipse.jetty.webapp.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
+
+/**
+ * This class launches the web service using Hadoop HttpServer2 (which uses
+ * an embedded Jetty container). This is the entry point to your service.
+ * The Java command used to launch this app should call the main method.
+ */
+public class ApiServerWebApp extends AbstractService {
+ private static final Logger logger = LoggerFactory
+ .getLogger(ApiServerWebApp.class);
+ private static final String SEP = ";";
+
+ // REST API server for YARN native services
+ private HttpServer2 apiServer;
+ private InetSocketAddress bindAddress;
+
+ public static void main(String[] args) throws IOException {
+ ApiServerWebApp apiWebApp = new ApiServerWebApp();
+ try {
+ apiWebApp.startWebApp();
+ } catch (Exception e) {
+ apiWebApp.close();
+ }
+ }
+
+ public ApiServerWebApp() {
+ super(ApiServerWebApp.class.getName());
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS,
+ DEFAULT_API_SERVER_ADDRESS , DEFAULT_API_SERVER_PORT);
+ logger.info("YARN API server running on " + bindAddress);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ doSecureLogin(getConfig());
+ }
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (apiServer != null) {
+ apiServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ private void doSecureLogin(org.apache.hadoop.conf.Configuration conf)
+ throws IOException {
+ SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB,
+ YarnConfiguration.RM_PRINCIPAL, bindAddress.getHostName());
+ addFilters(conf);
+ }
+
+ private void addFilters(org.apache.hadoop.conf.Configuration conf) {
+ // Always load pseudo authentication filter to parse "user.name" in an URL
+ // to identify a HTTP request's user.
+ boolean hasHadoopAuthFilterInitializer = false;
+ String filterInitializerConfKey = "hadoop.http.filter.initializers";
+ Class<?>[] initializersClasses =
+ conf.getClasses(filterInitializerConfKey);
+ List<String> targets = new ArrayList<String>();
+ if (initializersClasses != null) {
+ for (Class<?> initializer : initializersClasses) {
+ if (initializer.getName().equals(
+ AuthenticationFilterInitializer.class.getName())) {
+ hasHadoopAuthFilterInitializer = true;
+ break;
+ }
+ targets.add(initializer.getName());
+ }
+ }
+ if (!hasHadoopAuthFilterInitializer) {
+ targets.add(AuthenticationFilterInitializer.class.getName());
+ conf.set(filterInitializerConfKey, StringUtils.join(",", targets));
+ }
+ }
+
+ private void startWebApp() throws IOException {
+ URI uri = URI.create("http://" + NetUtils.getHostPortString(bindAddress));
+
+ apiServer = new HttpServer2.Builder()
+ .setName("api-server")
+ .setConf(getConfig())
+ .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
+ .setUsernameConfKey(RM_WEBAPP_SPNEGO_USER_NAME_KEY)
+ .setKeytabConfKey(RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+ .addEndpoint(uri).build();
+
+ String apiPackages =
+ ApiServer.class.getPackage().getName() + SEP
+ + GenericExceptionHandler.class.getPackage().getName() + SEP
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
+ apiServer.addJerseyResourcePackage(apiPackages, "/*");
+
+ try {
+ logger.info("Service starting up. Logging start...");
+ apiServer.start();
+ logger.info("Server status = {}", apiServer.toString());
+ for (Configuration conf : apiServer.getWebAppContext()
+ .getConfigurations()) {
+ logger.info("Configurations = {}", conf);
+ }
+ logger.info("Context Path = {}", Collections.singletonList(
+ apiServer.getWebAppContext().getContextPath()));
+ logger.info("ResourceBase = {}", Collections.singletonList(
+ apiServer.getWebAppContext().getResourceBase()));
+ logger.info("War = {}", Collections
+ .singletonList(apiServer.getWebAppContext().getWar()));
+ } catch (Exception ex) {
+ logger.error("Hadoop HttpServer2 App **failed**", ex);
+ throw ex;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
deleted file mode 100644
index 30fc5ef..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.webapp;
-
-import com.google.inject.Singleton;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ApplicationState;
-import org.apache.hadoop.yarn.service.api.records.ApplicationStatus;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
-
-@Singleton
-@Path(CONTEXT_ROOT)
-public class ApplicationApiService {
- private static final Logger LOG =
- LoggerFactory.getLogger(ApplicationApiService.class);
- private static Configuration YARN_CONFIG = new YarnConfiguration();
- private static ServiceClient SERVICE_CLIENT;
-
- static {
- init();
- }
-
- // initialize all the common resources - order is important
- private static void init() {
- SERVICE_CLIENT = new ServiceClient();
- SERVICE_CLIENT.init(YARN_CONFIG);
- SERVICE_CLIENT.start();
- }
-
- @GET
- @Path(VERSION)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
- public Response getVersion() {
- String version = VersionInfo.getBuildVersion();
- LOG.info(version);
- return Response.ok(version).build();
- }
-
- @POST
- @Path(APP_ROOT_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response createApplication(Application application) {
- LOG.info("POST: createApplication = {}", application);
- ApplicationStatus applicationStatus = new ApplicationStatus();
- try {
- ApplicationId applicationId = SERVICE_CLIENT.actionCreate(application);
- LOG.info("Successfully created application " + application.getName()
- + " applicationId = " + applicationId);
- applicationStatus.setState(ApplicationState.ACCEPTED);
- applicationStatus.setUri(
- CONTEXT_ROOT + APP_ROOT_PATH + "/" + application
- .getName());
- return Response.status(Status.CREATED).entity(applicationStatus).build();
- } catch (IllegalArgumentException e) {
- applicationStatus.setDiagnostics(e.getMessage());
- return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
- .build();
- } catch (Exception e) {
- String message = "Failed to create application " + application.getName();
- LOG.error(message, e);
- applicationStatus.setDiagnostics(message + ": " + e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
- }
- }
-
- @GET
- @Path(APP_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response getApplication(@PathParam(APP_NAME) String appName) {
- LOG.info("GET: getApplication for appName = {}", appName);
- ApplicationStatus applicationStatus = new ApplicationStatus();
-
- // app name validation
- if (!SliderUtils.isClusternameValid(appName)) {
- applicationStatus.setDiagnostics("Invalid application name: " + appName);
- applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
- return Response.status(Status.NOT_FOUND).entity(applicationStatus)
- .build();
- }
-
- try {
- Application app = SERVICE_CLIENT.getStatus(appName);
- return Response.ok(app).build();
- } catch (Exception e) {
- LOG.error("Get application failed", e);
- applicationStatus
- .setDiagnostics("Failed to retrieve application: " + e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
- }
- }
-
- @DELETE
- @Path(APP_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response deleteApplication(@PathParam(APP_NAME) String appName) {
- LOG.info("DELETE: deleteApplication for appName = {}", appName);
- return stopApplication(appName, true);
- }
-
- private Response stopApplication(String appName, boolean destroy) {
- try {
- SERVICE_CLIENT.actionStop(appName, destroy);
- if (destroy) {
- SERVICE_CLIENT.actionDestroy(appName);
- LOG.info("Successfully deleted application {}", appName);
- } else {
- LOG.info("Successfully stopped application {}", appName);
- }
- return Response.status(Status.NO_CONTENT).build();
- } catch (ApplicationNotFoundException e) {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- applicationStatus.setDiagnostics(
- "Application " + appName + " not found " + e.getMessage());
- return Response.status(Status.NOT_FOUND).entity(applicationStatus)
- .build();
- } catch (Exception e) {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- applicationStatus.setDiagnostics(e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
- }
- }
-
- @PUT
- @Path(COMPONENT_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
- public Response updateComponent(@PathParam(APP_NAME) String appName,
- @PathParam(COMPONENT_NAME) String componentName, Component component) {
-
- if (component.getNumberOfContainers() < 0) {
- return Response.status(Status.BAD_REQUEST).entity(
- "Application = " + appName + ", Component = " + component.getName()
- + ": Invalid number of containers specified " + component
- .getNumberOfContainers()).build();
- }
- try {
- Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
- Collections.singletonMap(component.getName(),
- component.getNumberOfContainers()));
- return Response.ok().entity(
- "Updating component " + componentName + " size from " + original
- .get(componentName) + " to " + component.getNumberOfContainers())
- .build();
- } catch (YarnException | IOException e) {
- ApplicationStatus status = new ApplicationStatus();
- status.setDiagnostics(e.getMessage());
- return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
- .build();
- }
- }
-
- @PUT
- @Path(APP_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON })
- public Response updateApplication(@PathParam(APP_NAME) String appName,
- Application updateAppData) {
- LOG.info("PUT: updateApplication for app = {} with data = {}", appName,
- updateAppData);
-
- // Ignore the app name provided in updateAppData and always use appName
- // path param
- updateAppData.setName(appName);
-
- // For STOP the app should be running. If already stopped then this
- // operation will be a no-op. For START it should be in stopped state.
- // If already running then this operation will be a no-op.
- if (updateAppData.getState() != null
- && updateAppData.getState() == ApplicationState.STOPPED) {
- return stopApplication(appName, false);
- }
-
- // If a START is requested
- if (updateAppData.getState() != null
- && updateAppData.getState() == ApplicationState.STARTED) {
- return startApplication(appName);
- }
-
- // If new lifetime value specified then update it
- if (updateAppData.getLifetime() != null
- && updateAppData.getLifetime() > 0) {
- return updateLifetime(appName, updateAppData);
- }
-
- // flex a single component app
- if (updateAppData.getNumberOfContainers() != null && !ServiceApiUtil
- .hasComponent(updateAppData)) {
- Component defaultComp = ServiceApiUtil.createDefaultComponent(updateAppData);
- return updateComponent(updateAppData.getName(), defaultComp.getName(),
- defaultComp);
- }
-
- // If nothing happens consider it a no-op
- return Response.status(Status.NO_CONTENT).build();
- }
-
- private Response updateLifetime(String appName, Application updateAppData) {
- try {
- String newLifeTime =
- SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
- return Response.ok("Application " + appName + " lifeTime is successfully updated to "
- + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
- } catch (Exception e) {
- String message =
- "Failed to update application (" + appName + ") lifetime ("
- + updateAppData.getLifetime() + ")";
- LOG.error(message, e);
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + " : " + e.getMessage()).build();
- }
- }
-
- private Response startApplication(String appName) {
- try {
- SERVICE_CLIENT.actionStart(appName);
- LOG.info("Successfully started application " + appName);
- return Response.ok("Application " + appName + " is successfully started").build();
- } catch (Exception e) {
- String message = "Failed to start application " + appName;
- LOG.info(message, e);
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + ": " + e.getMessage()).build();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
deleted file mode 100644
index 7225209..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.webapp;
-
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URI;
-import java.util.Arrays;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.eclipse.jetty.webapp.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class launches the web application using Hadoop HttpServer2 (which uses
- * an embedded Jetty container). This is the entry point to your application.
- * The Java command used to launch this app should call the main method.
- */
-public class ApplicationApiWebApp extends AbstractService {
- private static final Logger logger = LoggerFactory
- .getLogger(ApplicationApiWebApp.class);
- private static final String SEP = ";";
-
- // REST API server for YARN native services
- private HttpServer2 applicationApiServer;
-
- public static void main(String[] args) throws IOException {
- ApplicationApiWebApp apiWebApp = new ApplicationApiWebApp();
- try {
- apiWebApp.startWebApp();
- } catch (Exception e) {
- if (apiWebApp != null) {
- apiWebApp.close();
- }
- }
- }
-
- public ApplicationApiWebApp() {
- super(ApplicationApiWebApp.class.getName());
- }
-
- @Override
- protected void serviceStart() throws Exception {
- startWebApp();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (applicationApiServer != null) {
- applicationApiServer.stop();
- }
- super.serviceStop();
- }
-
- protected void startWebApp() throws IOException {
- // The port that we should run on can be set into an environment variable
- // Look for that variable and default to 9191 if it isn't there.
- String webPort = System.getenv(PROPERTY_REST_SERVICE_PORT);
- if (StringUtils.isEmpty(webPort)) {
- webPort = "9191";
- }
-
- String webHost = System.getenv(PROPERTY_REST_SERVICE_HOST);
- if (StringUtils.isEmpty(webHost)) {
- webHost = InetAddress.getLocalHost().getHostName();
- }
- logger.info("YARN native services REST API running on host {} and port {}",
- webHost, webPort);
- logger.info("Configuration = {}", getConfig());
-
- applicationApiServer = new HttpServer2.Builder()
- .setName("services-rest-api")
- .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
-
- String apiPackages =
- ApplicationApiService.class.getPackage().getName() + SEP
- + GenericExceptionHandler.class.getPackage().getName() + SEP
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
- applicationApiServer.addJerseyResourcePackage(apiPackages, "/*");
-
- try {
- logger.info("Application starting up. Logging start...");
- applicationApiServer.start();
- logger.info("Server status = {}", applicationApiServer.toString());
- for (Configuration conf : applicationApiServer.getWebAppContext()
- .getConfigurations()) {
- logger.info("Configurations = {}", conf);
- }
- logger.info("Context Path = {}", Arrays.asList(applicationApiServer
- .getWebAppContext().getContextPath()));
- logger.info("ResourceBase = {}", Arrays.asList(applicationApiServer
- .getWebAppContext().getResourceBase()));
- logger.info("War = {}",
- Arrays.asList(applicationApiServer.getWebAppContext().getWar()));
- } catch (Exception ex) {
- logger.error("Hadoop HttpServer2 App **failed**", ex);
- throw ex;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/58] [abbrv] hadoop git commit: YARN-7323. Data structure update
in service REST API. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7323. Data structure update in service REST API. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68acd88d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68acd88d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68acd88d
Branch: refs/heads/yarn-native-services
Commit: 68acd88dcbfe03a0134c60b5398dfaa31ad2b786
Parents: 3c30b1a
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Oct 17 11:50:16 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:18 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/service/webapp/ApiServer.java | 9 ---
...RN-Simplified-V1-API-Layer-For-Services.yaml | 67 ++++++++---------
.../hadoop/yarn/service/ServiceScheduler.java | 1 +
.../yarn/service/api/records/Component.java | 22 +++++-
.../service/api/records/ComponentState.java | 30 ++++++++
.../yarn/service/api/records/Container.java | 15 ++--
.../yarn/service/api/records/Service.java | 76 --------------------
.../yarn/service/api/records/ServiceState.java | 2 +-
.../yarn/service/client/ServiceClient.java | 45 ++++++++----
.../yarn/service/component/Component.java | 12 ++++
.../component/instance/ComponentInstance.java | 48 +++++++++----
.../yarn/service/conf/RestApiConstants.java | 4 --
.../yarn/service/utils/ServiceApiUtil.java | 53 ++------------
.../hadoop/yarn/service/ServiceTestUtils.java | 4 +-
.../hadoop/yarn/service/TestServiceApiUtil.java | 74 ++++++-------------
.../yarn/service/TestYarnNativeServices.java | 4 +-
.../service/conf/examples/app-override.json | 8 ++-
.../hadoop/yarn/service/conf/examples/app.json | 8 ++-
.../yarn/service/conf/examples/external0.json | 15 ++--
.../markdown/yarn-service/YarnServiceAPI.md | 22 +++---
20 files changed, 232 insertions(+), 287 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 5773069..1bb6c93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -242,15 +242,6 @@ public class ApiServer {
return updateLifetime(appName, updateServiceData);
}
- // flex a single component app
- if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil
- .hasComponent(updateServiceData)) {
- Component defaultComp = ServiceApiUtil
- .createDefaultComponent(updateServiceData);
- return updateComponent(updateServiceData.getName(), defaultComp.getName(),
- defaultComp);
- }
-
// If nothing happens consider it a no-op
return Response.status(Status.NO_CONTENT).build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index b9b5b3a..cc76259 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -19,28 +19,17 @@ swagger: '2.0'
info:
title: "YARN Simplified API layer for services"
description: |
- Bringing a new service on YARN today is not a simple experience. The APIs of
- existing frameworks are either too low level (native YARN), require writing
- new code (for frameworks with programmatic APIs) or writing a complex spec
- (for declarative frameworks). In addition to building critical building blocks
- inside YARN (as part of other efforts at YARN-4692), there is a need for
- simplifying the user facing story for building services. Experience of projects
- like Apache Slider running real-life services like HBase, Storm, Accumulo,
- Solr etc, gives us some very good insights on how simplified APIs for services
- should look like.
+ Bringing a new service on YARN today is not a simple experience. The APIs of existing
+ frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs)
+ or writing a complex spec (for declarative frameworks).
- To this end, we should look at a new simple-services API layer backed by REST
- interfaces. This API can be used to create and manage the lifecycle of YARN
- services. Services here can range from simple single-component service to
- complex multi-component assemblies needing orchestration. YARN-4793 tracks
- this effort.
+ This simplified REST API can be used to create and manage the lifecycle of YARN services.
+ In most cases, the application owner will not be forced to make any changes to their applications.
+ This is primarily true if the application is packaged with containerization technologies like Docker.
- This document spotlights on this specification. In most of the cases, the
- application owner will not be forced to make any changes to their applications.
- This is primarily true if the application is packaged with containerization
- technologies like docker. Irrespective of how complex the application is,
- there will be hooks provided at appropriate layers to allow pluggable and
- customizable application behavior.
+ This document describes the API specifications (aka. YarnFile) for deploying/managing
+ containerized services on YARN. The same JSON spec can be used for both REST API
+ and CLI to manage the services.
version: "1.0.0"
license:
@@ -216,22 +205,15 @@ definitions:
type: string
description: A unique service id.
artifact:
- description: Artifact of single-component service.
+ description: The default artifact for all components of the service except the components which has Artifact type set to SERVICE (optional).
$ref: '#/definitions/Artifact'
resource:
- description: Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.
+ description: The default resource for all components of the service (optional).
$ref: '#/definitions/Resource'
- launch_command:
- type: string
- description: The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.
launch_time:
type: string
format: date
description: The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
- number_of_containers:
- type: integer
- format: int64
- description: Number of containers for each component in the service. Each component can further override this service-level global default.
number_of_running_containers:
type: integer
format: int64
@@ -251,13 +233,8 @@ definitions:
configuration:
description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.
$ref: '#/definitions/Configuration'
- containers:
- description: Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.
- type: array
- items:
- $ref: '#/definitions/Container'
state:
- description: State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.
+ description: State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state.
$ref: '#/definitions/ServiceState'
quicklinks:
type: object
@@ -314,6 +291,9 @@ definitions:
name:
type: string
description: Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
+ state:
+ description: The state of the component
+ $ref: "#/definitions/ComponentState"
dependencies:
type: array
items:
@@ -431,9 +411,11 @@ definitions:
state:
description: State of the container of a service.
$ref: '#/definitions/ContainerState'
- component_name:
+ component_instance_name:
type: string
- description: Name of the component that this container instance belongs to.
+ description: Name of the component instance that this container instance belongs to. Component instance name is named as $COMPONENT_NAME-i, where i is a
+ monotonically increasing integer. E.g. A componet called nginx can have multiple component instances named as nginx-0, nginx-1 etc.
+ Each component instance is backed by a container instance.
resource:
description: Resource used for this container.
$ref: '#/definitions/Resource'
@@ -452,7 +434,7 @@ definitions:
enum:
- ACCEPTED
- STARTED
- - READY
+ - STABLE
- STOPPED
- FAILED
ContainerState:
@@ -465,6 +447,15 @@ definitions:
- INIT
- STARTED
- READY
+ ComponentState:
+ description: The state of the component
+ properties:
+ state:
+ type: string
+ description: enum of the state of the component
+ enum:
+ - FLEXING
+ - STABLE
ServiceStatus:
description: The current status of a submitted service, returned as a response to the GET API.
properties:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index f3824df..631f89e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -604,6 +604,7 @@ public class ServiceScheduler extends CompositeService {
LOG.error("No component instance exists for " + containerId);
return;
}
+ LOG.error("Failed to start " + containerId, t);
amRMClient.releaseAssignedContainer(containerId);
// After container released, it'll get CONTAINER_COMPLETED event from RM
// automatically which will trigger stopping COMPONENT INSTANCE
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
index b0a8e82..fe9c043 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
@@ -59,6 +59,7 @@ public class Component implements Serializable {
private Long numberOfContainers = null;
private Boolean runPrivilegedContainer = false;
private PlacementPolicy placementPolicy = null;
+ private ComponentState state = ComponentState.FLEXING;
private Configuration configuration = new Configuration();
private List<String> quicklinks = new ArrayList<String>();
private List<Container> containers =
@@ -305,6 +306,21 @@ public class Component implements Serializable {
this.quicklinks = quicklinks;
}
+ public Component state(ComponentState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "State of the component.")
+ @JsonProperty("state")
+ public ComponentState getState() {
+ return state;
+ }
+
+ public void setState(ComponentState state) {
+ this.state = state;
+ }
+
@Override
public boolean equals(java.lang.Object o) {
if (this == o) {
@@ -325,14 +341,15 @@ public class Component implements Serializable {
component.runPrivilegedContainer)
&& Objects.equals(this.placementPolicy, component.placementPolicy)
&& Objects.equals(this.configuration, component.configuration)
- && Objects.equals(this.quicklinks, component.quicklinks);
+ && Objects.equals(this.quicklinks, component.quicklinks)
+ && Objects.equals(this.state, component.state);
}
@Override
public int hashCode() {
return Objects.hash(name, dependencies, readinessCheck, artifact,
launchCommand, resource, numberOfContainers,
- runPrivilegedContainer, placementPolicy, configuration, quicklinks);
+ runPrivilegedContainer, placementPolicy, configuration, quicklinks, state);
}
@Override
@@ -341,6 +358,7 @@ public class Component implements Serializable {
sb.append("class Component {\n");
sb.append(" name: ").append(toIndentedString(name)).append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
sb.append(" dependencies: ").append(toIndentedString(dependencies))
.append("\n");
sb.append(" readinessCheck: ").append(toIndentedString(readinessCheck))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentState.java
new file mode 100644
index 0000000..702a9ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentState.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "The current state of a component.")
+public enum ComponentState {
+ FLEXING, STABLE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
index cf8cd79..af06542 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
@@ -49,7 +49,7 @@ public class Container extends BaseResource {
private String hostname = null;
private String bareHost = null;
private ContainerState state = null;
- private String componentName = null;
+ private String componentInstanceName = null;
private Resource resource = null;
private Artifact artifact = null;
private Boolean privilegedContainer = null;
@@ -176,19 +176,19 @@ public class Container extends BaseResource {
* Name of the component that this container instance belongs to.
**/
public Container componentName(String componentName) {
- this.componentName = componentName;
+ this.componentInstanceName = componentName;
return this;
}
@ApiModelProperty(example = "null", value = "Name of the component that this container instance belongs to.")
@JsonProperty("component_name")
- public String getComponentName() {
- return componentName;
+ public String getComponentInstanceName() {
+ return componentInstanceName;
}
@XmlElement(name = "component_name")
- public void setComponentName(String componentName) {
- this.componentName = componentName;
+ public void setComponentInstanceName(String componentInstanceName) {
+ this.componentInstanceName = componentInstanceName;
}
/**
@@ -274,7 +274,8 @@ public class Container extends BaseResource {
sb.append(" hostname: ").append(toIndentedString(hostname)).append("\n");
sb.append(" bareHost: ").append(toIndentedString(bareHost)).append("\n");
sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" componentName: ").append(toIndentedString(componentName))
+ sb.append(" componentInstanceName: ").append(toIndentedString(
+ componentInstanceName))
.append("\n");
sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
index f3bbaa0..77a2610 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
@@ -53,15 +53,12 @@ public class Service extends BaseResource {
private String id = null;
private Artifact artifact = null;
private Resource resource = null;
- private String launchCommand = null;
private Date launchTime = null;
- private Long numberOfContainers = null;
private Long numberOfRunningContainers = null;
private Long lifetime = null;
private PlacementPolicy placementPolicy = null;
private List<Component> components = new ArrayList<>();
private Configuration configuration = new Configuration();
- private List<Container> containers = new ArrayList<>();
private ServiceState state = null;
private Map<String, String> quicklinks = new HashMap<>();
private String queue = null;
@@ -143,29 +140,6 @@ public class Service extends BaseResource {
}
/**
- * The custom launch command of an service component (optional). If not
- * specified for services with docker images say, it will default to the
- * default start command of the image. If there is a single component in this
- * service, you can specify this without the need to have a 'components'
- * section.
- **/
- public Service launchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The custom launch command of an service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.")
- @JsonProperty("launch_command")
- public String getLaunchCommand() {
- return launchCommand;
- }
-
- @XmlElement(name = "launch_command")
- public void setLaunchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- }
-
- /**
* The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
**/
public Service launchTime(Date launchTime) {
@@ -185,26 +159,6 @@ public class Service extends BaseResource {
}
/**
- * Number of containers for each component in the service. Each
- * component can further override this service-level global default.
- **/
- public Service numberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Number of containers for each component in the service. Each component can further override this service-level global default.")
- @JsonProperty("number_of_containers")
- public Long getNumberOfContainers() {
- return numberOfContainers;
- }
-
- @XmlElement(name = "number_of_containers")
- public void setNumberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- }
-
- /**
* In get response this provides the total number of running containers for
* this service (across all components) at the time of request. Note, a
* subsequent request can return a different number as and when more
@@ -323,30 +277,6 @@ public class Service extends BaseResource {
}
/**
- * Containers of a started service. Specifying a value for this attribute
- * for the POST payload raises a validation error. This blob is available only
- * in the GET response of a started service.
- **/
- public Service containers(List<Container> containers) {
- this.containers = containers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
- @JsonProperty("containers")
- public List<Container> getContainers() {
- return containers;
- }
-
- public void setContainers(List<Container> containers) {
- this.containers = containers;
- }
-
- public void addContainer(Container container) {
- this.containers.add(container);
- }
-
- /**
* State of the service. Specifying a value for this attribute for the
* POST payload raises a validation error. This attribute is available only in
* the GET response of a started service.
@@ -428,12 +358,8 @@ public class Service extends BaseResource {
sb.append(" id: ").append(toIndentedString(id)).append("\n");
sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
- .append("\n");
sb.append(" launchTime: ").append(toIndentedString(launchTime))
.append("\n");
- sb.append(" numberOfContainers: ")
- .append(toIndentedString(numberOfContainers)).append("\n");
sb.append(" numberOfRunningContainers: ")
.append(toIndentedString(numberOfRunningContainers)).append("\n");
sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n");
@@ -443,8 +369,6 @@ public class Service extends BaseResource {
.append("\n");
sb.append(" configuration: ").append(toIndentedString(configuration))
.append("\n");
- sb.append(" containers: ").append(toIndentedString(containers))
- .append("\n");
sb.append(" state: ").append(toIndentedString(state)).append("\n");
sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
.append("\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
index a4509bd..d2f5d06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
@@ -29,5 +29,5 @@ import org.apache.hadoop.classification.InterfaceStability;
@ApiModel(description = "The current state of an service.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public enum ServiceState {
- ACCEPTED, STARTED, READY, STOPPED, FAILED;
+ ACCEPTED, STARTED, STABLE, STOPPED, FAILED;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index a3a9fd0..f6d35e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -62,8 +62,8 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.ServiceMaster;
-import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
@@ -73,23 +73,23 @@ import org.apache.hadoop.yarn.service.client.params.Arguments;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.client.params.CommonArgs;
import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
-import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
-import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -196,11 +196,7 @@ public class ServiceClient extends CompositeService
serviceDef = loadAppJsonFromLocalFS(args);
} else if (!StringUtils.isEmpty(args.example)) {
// create an example service
- String yarnHome = System
- .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
- args.file = new File(MessageFormat
- .format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
- yarnHome, args.example, args.example));
+ args.file = findExampleService(args);
serviceDef = loadAppJsonFromLocalFS(args);
} else {
throw new YarnException("No service definition provided!");
@@ -209,6 +205,27 @@ public class ServiceClient extends CompositeService
return EXIT_SUCCESS;
}
+ private File findExampleService(ActionCreateArgs args) throws YarnException {
+ String yarnHome = System
+ .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
+ // First look for the standard location.
+ File file = new File(MessageFormat
+ .format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
+ yarnHome, args.example, args.example));
+ if (file.exists()) {
+ return file;
+ }
+ // Then look for secondary location.
+ file = new File(MessageFormat
+ .format("{0}/yarn-service-examples/{1}/{2}.json", yarnHome,
+ args.example, args.example));
+ if (file.exists()) {
+ return file;
+ }
+ throw new YarnException(
+ "Example service " + args.example + " does not exist!");
+ }
+
public ApplicationId actionCreate(Service service)
throws IOException, YarnException {
String serviceName = service.getName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 98bb238..a5dd39c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -202,6 +202,8 @@ public class Component implements EventHandler<ComponentEvent> {
+ before + " to " + event.getDesired());
component.requestContainers(delta);
component.createNumCompInstances(delta);
+ component.componentSpec.setState(
+ org.apache.hadoop.yarn.service.api.records.ComponentState.FLEXING);
return FLEXING;
} else if (delta < 0){
delta = 0 - delta;
@@ -224,10 +226,14 @@ public class Component implements EventHandler<ComponentEvent> {
component.instanceIdCounter.decrementAndGet();
instance.destroy();
}
+ component.componentSpec.setState(
+ org.apache.hadoop.yarn.service.api.records.ComponentState.STABLE);
return STABLE;
} else {
LOG.info("[FLEX COMPONENT " + component.getName() + "]: already has " +
event.getDesired() + " instances, ignoring");
+ component.componentSpec.setState(
+ org.apache.hadoop.yarn.service.api.records.ComponentState.STABLE);
return STABLE;
}
}
@@ -297,8 +303,12 @@ public class Component implements EventHandler<ComponentEvent> {
// if desired == running
if (component.componentMetrics.containersRunning.value() == component
.getComponentSpec().getNumberOfContainers()) {
+ component.componentSpec.setState(
+ org.apache.hadoop.yarn.service.api.records.ComponentState.STABLE);
return STABLE;
} else {
+ component.componentSpec.setState(
+ org.apache.hadoop.yarn.service.api.records.ComponentState.FLEXING);
return FLEXING;
}
}
@@ -317,6 +327,8 @@ public class Component implements EventHandler<ComponentEvent> {
component.compInstanceDispatcher.getEventHandler().handle(
new ComponentInstanceEvent(event.getStatus().getContainerId(),
STOP).setStatus(event.getStatus()));
+ component.componentSpec.setState(
+ org.apache.hadoop.yarn.service.api.records.ComponentState.FLEXING);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 68c0537..6e585aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -99,6 +99,11 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
ComponentInstanceEventType, ComponentInstanceEvent>(INIT)
.addTransition(INIT, STARTED, START,
new ContainerStartedTransition())
+ .addTransition(INIT, INIT, STOP,
+ // container failed before launching, nothing to cleanup from registry
+ // This could happen if NMClient#startContainerAsync failed, container
+ // will be completed, but COMP_INSTANCE is still at INIT.
+ new ContainerStoppedTransition(true))
//From Running
.addTransition(STARTED, INIT, STOP,
@@ -159,7 +164,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
container.setLaunchTime(new Date(containerStartTime));
container.setState(ContainerState.RUNNING_BUT_UNREADY);
container.setBareHost(compInstance.container.getNodeId().getHost());
- container.setComponentName(compInstance.getCompInstanceName());
+ container.setComponentInstanceName(compInstance.getCompInstanceName());
if (compInstance.containerSpec != null) {
// remove the previous container.
compInstance.getCompSpec().removeContainer(compInstance.containerSpec);
@@ -194,6 +199,16 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
private static class ContainerStoppedTransition extends BaseTransition {
+ // whether the container failed before launched by AM or not.
+ boolean failedBeforeLaunching = false;
+ public ContainerStoppedTransition(boolean failedBeforeLaunching) {
+ this.failedBeforeLaunching = failedBeforeLaunching;
+ }
+
+ public ContainerStoppedTransition() {
+ this(false);
+ }
+
@Override
public void transition(ComponentInstance compInstance,
ComponentInstanceEvent event) {
@@ -225,24 +240,27 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
shouldExit = true;
}
- // clean up registry
- // hdfs dir content will be overwritten when a new container gets started,
- // so no need remove.
- compInstance.scheduler.executorService
- .submit(compInstance::cleanupRegistry);
+ if (!failedBeforeLaunching) {
+ // clean up registry
+ // If the container failed before launching, no need to cleanup registry,
+ // because it was not registered before.
+ // hdfs dir content will be overwritten when a new container gets started,
+ // so no need remove.
+ compInstance.scheduler.executorService
+ .submit(compInstance::cleanupRegistry);
+ if (compInstance.timelineServiceEnabled) {
+ // record in ATS
+ compInstance.serviceTimelinePublisher
+ .componentInstanceFinished(compInstance,
+ event.getStatus().getExitStatus(), event.getStatus().getState(),
+ containerDiag);
+ }
+ compInstance.containerSpec.setState(ContainerState.STOPPED);
+ }
// remove the failed ContainerId -> CompInstance mapping
comp.getScheduler().removeLiveCompInstance(event.getContainerId());
- if (compInstance.timelineServiceEnabled) {
- // record in ATS
- compInstance.serviceTimelinePublisher
- .componentInstanceFinished(compInstance,
- event.getStatus().getExitStatus(), event.getStatus().getState(),
- containerDiag);
- }
-
- compInstance.containerSpec.setState(ContainerState.STOPPED);
if (shouldExit) {
// Sleep for 5 seconds in hope that the state can be recorded in ATS.
// in case there's a client polling the comp state, it can be notified.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
index 6de2dc0..35e1980 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
@@ -30,10 +30,6 @@ public interface RestApiConstants {
String SERVICE_NAME = "service_name";
String COMPONENT_NAME = "component_name";
- String DEFAULT_COMPONENT_NAME = "default";
-
- String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
- String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
Long DEFAULT_UNLIMITED_LIFETIME = -1l;
Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index de82580..68db0bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -80,38 +80,14 @@ public class ServiceApiUtil {
validateNameFormat(service.getName(), conf);
- // If the service has no components do top-level checks
+ // If the service has no components, throw error
if (!hasComponent(service)) {
- // If artifact is of type SERVICE, read other service components
- if (service.getArtifact() != null && service.getArtifact()
- .getType() == Artifact.TypeEnum.SERVICE) {
- if (StringUtils.isEmpty(service.getArtifact().getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- Service otherService = loadService(fs,
- service.getArtifact().getId());
- service.setComponents(otherService.getComponents());
- service.setArtifact(null);
- SliderUtils.mergeMapsIgnoreDuplicateKeys(service.getQuicklinks(),
- otherService.getQuicklinks());
- } else {
- // Since it is a simple service with no components, create a default
- // component
- Component comp = createDefaultComponent(service);
- validateComponent(comp, fs.getFileSystem(), conf);
- service.getComponents().add(comp);
- if (service.getLifetime() == null) {
- service.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
- }
- return;
- }
+ throw new IllegalArgumentException(
+ "No component specified for " + service.getName());
}
// Validate there are no component name collisions (collisions are not
// currently supported) and add any components from external services
- // TODO allow name collisions? see AppState#roles
- // TODO or add prefix to external component names?
Configuration globalConf = service.getConfiguration();
Set<String> componentNames = new HashSet<>();
List<Component> componentsToRemove = new ArrayList<>();
@@ -174,8 +150,6 @@ public class ServiceApiUtil {
// values are not provided
Artifact globalArtifact = service.getArtifact();
Resource globalResource = service.getResource();
- Long globalNumberOfContainers = service.getNumberOfContainers();
- String globalLaunchCommand = service.getLaunchCommand();
for (Component comp : service.getComponents()) {
// fill in global artifact unless it is type SERVICE
if (comp.getArtifact() == null && service.getArtifact() != null
@@ -187,14 +161,6 @@ public class ServiceApiUtil {
if (comp.getResource() == null) {
comp.setResource(globalResource);
}
- // fill in global container count
- if (comp.getNumberOfContainers() == null) {
- comp.setNumberOfContainers(globalNumberOfContainers);
- }
- // fill in global launch command
- if (comp.getLaunchCommand() == null) {
- comp.setLaunchCommand(globalLaunchCommand);
- }
// validate dependency existence
if (comp.getDependencies() != null) {
for (String dependency : comp.getDependencies()) {
@@ -360,7 +326,7 @@ public class ServiceApiUtil {
}
}
- public static boolean hasComponent(Service service) {
+ private static boolean hasComponent(Service service) {
if (service.getComponents() == null || service.getComponents()
.isEmpty()) {
return false;
@@ -368,17 +334,6 @@ public class ServiceApiUtil {
return true;
}
- public static Component createDefaultComponent(Service service) {
- Component comp = new Component();
- comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME);
- comp.setArtifact(service.getArtifact());
- comp.setResource(service.getResource());
- comp.setNumberOfContainers(service.getNumberOfContainers());
- comp.setLaunchCommand(service.getLaunchCommand());
- comp.setConfiguration(service.getConfiguration());
- return comp;
- }
-
public static Collection<Component> sortByDependencies(List<Component>
components) {
Map<String, Component> sortedComponents =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index aeb4341..19a5177 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -84,11 +84,11 @@ public class ServiceTestUtils {
return exampleApp;
}
- protected Component createComponent(String name) {
+ public static Component createComponent(String name) {
return createComponent(name, 2L, "sleep 1000");
}
- protected Component createComponent(String name, long numContainers,
+ protected static Component createComponent(String name, long numContainers,
String command) {
Component comp1 = new Component();
comp1.setNumberOfContainers(numContainers);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
index 959e4d6..55c096e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -20,11 +20,11 @@ package org.apache.hadoop.yarn.service;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.Assert;
@@ -36,9 +36,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
import static org.junit.Assert.assertEquals;
@@ -99,6 +99,8 @@ public class TestServiceApiUtil {
// launch command not specified
app.setName(LEN_64_STR);
+ Component comp = new Component().name("comp1");
+ app.addComponent(comp);
try {
ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DEFAULT_DNS);
Assert.fail(EXCEPTION_PREFIX + "service with no launch command");
@@ -118,18 +120,8 @@ public class TestServiceApiUtil {
e.getMessage());
}
- // resource not specified
- app.setLaunchCommand("sleep 3600");
- try {
- ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "service with no resource");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
// memory not specified
+ comp.setLaunchCommand("sleep 1");
Resource res = new Resource();
app.setResource(res);
try {
@@ -138,7 +130,7 @@ public class TestServiceApiUtil {
} catch (IllegalArgumentException e) {
assertEquals(String.format(
RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
- DEFAULT_COMPONENT_NAME), e.getMessage());
+ comp.getName()), e.getMessage());
}
// invalid no of cpus
@@ -151,7 +143,7 @@ public class TestServiceApiUtil {
} catch (IllegalArgumentException e) {
assertEquals(String.format(
RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
- DEFAULT_COMPONENT_NAME), e.getMessage());
+ comp.getName()), e.getMessage());
}
// number of containers not specified
@@ -173,7 +165,7 @@ public class TestServiceApiUtil {
} catch (IllegalArgumentException e) {
assertEquals(String.format(RestApiErrorMessages
.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
- DEFAULT_COMPONENT_NAME),
+ comp.getName()),
e.getMessage());
}
@@ -202,25 +194,6 @@ public class TestServiceApiUtil {
Assert.assertTrue(e.getMessage()
.startsWith(ERROR_CONTAINERS_COUNT_INVALID));
}
-
- // negative number of containers
- app.setNumberOfContainers(-1L);
- try {
- ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // everything valid here
- app.setNumberOfContainers(5L);
- try {
- ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- LOG.error("service attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
}
@Test
@@ -228,15 +201,17 @@ public class TestServiceApiUtil {
SliderFileSystem sfs = ServiceTestUtils.initMockFs();
Service app = new Service();
- app.setName("name");
+ app.setName("service1");
Resource res = new Resource();
app.setResource(res);
res.setMemory("512M");
- app.setNumberOfContainers(3L);
// no artifact id fails with default type
Artifact artifact = new Artifact();
app.setArtifact(artifact);
+ Component comp = ServiceTestUtils.createComponent("comp1");
+
+ app.setComponents(Collections.singletonList(comp));
try {
ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "service with no artifact id");
@@ -272,9 +247,6 @@ public class TestServiceApiUtil {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
- // defaults assigned
- assertEquals(app.getComponents().get(0).getName(),
- DEFAULT_COMPONENT_NAME);
assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
}
@@ -289,15 +261,14 @@ public class TestServiceApiUtil {
comp.setName(compName);
comp.setResource(createValidResource());
comp.setNumberOfContainers(1L);
+ comp.setLaunchCommand("sleep 1");
return comp;
}
private static Service createValidApplication(String compName) {
Service app = new Service();
- app.setLaunchCommand("sleep 3600");
app.setName("name");
app.setResource(createValidResource());
- app.setNumberOfContainers(1L);
if (compName != null) {
app.addComponent(createValidComponent(compName));
}
@@ -315,7 +286,7 @@ public class TestServiceApiUtil {
artifact.setType(Artifact.TypeEnum.SERVICE);
artifact.setId("id");
app.setArtifact(artifact);
-
+ app.addComponent(ServiceTestUtils.createComponent("comp2"));
try {
ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
@@ -323,7 +294,7 @@ public class TestServiceApiUtil {
}
assertEquals(1, app.getComponents().size());
- assertNotNull(app.getComponent("comp1"));
+ assertNotNull(app.getComponent("comp2"));
}
@Test
@@ -410,12 +381,13 @@ public class TestServiceApiUtil {
@Test
public void testDependencySorting() throws IOException {
- Component a = new Component().name("a");
- Component b = new Component().name("b");
- Component c = new Component().name("c");
- Component d = new Component().name("d").dependencies(Arrays.asList("c"));
- Component e = new Component().name("e").dependencies(Arrays.asList("b",
- "d"));
+ Component a = ServiceTestUtils.createComponent("a");
+ Component b = ServiceTestUtils.createComponent("b");
+ Component c = ServiceTestUtils.createComponent("c");
+ Component d =
+ ServiceTestUtils.createComponent("d").dependencies(Arrays.asList("c"));
+ Component e = ServiceTestUtils.createComponent("e")
+ .dependencies(Arrays.asList("b", "d"));
verifyDependencySorting(Arrays.asList(a, b, c), a, b, c);
verifyDependencySorting(Arrays.asList(c, a, b), c, a, b);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index d821b84..64fcf57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -176,7 +176,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
for (String comp : compOrder) {
long num = retrievedApp.getComponent(comp).getNumberOfContainers();
for (int i = 0; i < num; i++) {
- String compInstanceName = containerList.get(index).getComponentName();
+ String compInstanceName = containerList.get(index).getComponentInstanceName();
String compName =
compInstanceName.substring(0, compInstanceName.lastIndexOf('-'));
Assert.assertEquals(comp, compName);
@@ -219,7 +219,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
Assert.assertEquals(expectedNumInstances, component.getContainers().size());
TreeSet<String> instances = new TreeSet<>();
for (Container container : component.getContainers()) {
- instances.add(container.getComponentName());
+ instances.add(container.getComponentInstanceName());
}
int i = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
index d7e2fd0..753a9cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
@@ -1,7 +1,6 @@
{
"name": "app-1",
"lifetime": "3600",
- "launch_command": "sleep 3600",
"configuration": {
"properties": {
"g1": "a",
@@ -29,10 +28,11 @@
"cpus": 1,
"memory": "512"
},
- "number_of_containers": 2,
"components": [
{
"name": "simple",
+ "launch_command": "sleep 3600",
+ "number_of_containers": 2,
"configuration": {
"files": [
{
@@ -47,6 +47,8 @@
},
{
"name": "master",
+ "launch_command": "sleep 3600",
+ "number_of_containers": 2,
"configuration": {
"properties": {
"name": "m",
@@ -56,6 +58,8 @@
},
{
"name": "worker",
+ "number_of_containers": 2,
+ "launch_command": "sleep 3600",
"resource": {
"cpus": 1,
"memory": "1024"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
index 43d3c39..2eb477f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
@@ -2,7 +2,6 @@
"name": "app-1",
"id" : "application_1503358878042_0011",
"lifetime": "3600",
- "launch_command": "sleep 3600",
"configuration": {
"properties": {
"g1": "a",
@@ -14,14 +13,16 @@
"cpus": 1,
"memory": "512"
},
- "number_of_containers": 2,
"components": [
{
- "name": "simple"
+ "name": "simple",
+ "number_of_containers": 2,
+ "launch_command": "sleep 3600"
},
{
"name": "master",
"number_of_containers": 1,
+ "launch_command": "sleep 3600",
"configuration": {
"properties": {
"g1": "overridden",
@@ -33,6 +34,7 @@
{
"name": "worker",
"number_of_containers": 5,
+ "launch_command": "sleep 3600",
"resource": {
"cpus": 1,
"memory": "1024"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
index 0857f62..f0163bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
@@ -1,8 +1,15 @@
{
"name": "external-0",
"lifetime": "3600",
- "artifact": {
- "type": "SERVICE",
- "id": "app-1"
- }
+
+ "components" : [
+ {
+ "name" : "comp1",
+ "artifact": {
+ "type": "SERVICE",
+ "id": "app-1"
+ }
+ }
+ ]
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index 9022268..e9aaf7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -201,7 +201,6 @@ Set a component's desired number of instanes
|404|Service does not exist|No Content|
|default|Unexpected error|ServiceStatus|
-
## Definitions
### Artifact
@@ -221,6 +220,7 @@ One or more components of the service. If the service is HBase say, then the com
|Name|Description|Required|Schema|Default|
|----|----|----|----|----|
|name|Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.|true|string||
+|state|The state of the component|false|ComponentState||
|dependencies|An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.|false|string array||
|readiness_check|Readiness check for this component.|false|ReadinessCheck||
|artifact|Artifact of the component (optional). If not specified, the service level global artifact takes effect.|false|Artifact||
@@ -233,6 +233,15 @@ One or more components of the service. If the service is HBase say, then the com
|quicklinks|A list of quicklink keys defined at the service level, and to be resolved by this component.|false|string array||
+### ComponentState
+
+The state of the component
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|state|enum of the state of the component|false|enum (FLEXING, STABLE)||
+
+
### ConfigFile
A config file that needs to be created and made available as a volume in a service component container.
@@ -268,7 +277,7 @@ An instance of a running service container.
|hostname|Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
|bare_host|The bare node or host in which the container is running, e.g. cn008.example.com.|false|string||
|state|State of the container of a service.|false|ContainerState||
-|component_name|Name of the component that this container instance belongs to.|false|string||
+|component_instance_name|Name of the component instance that this container instance belongs to. Component instance name is named as $COMPONENT_NAME-i, where i is a monotonically increasing integer. E.g. A componet called nginx can have multiple component instances named as nginx-0, nginx-1 etc. Each component instance is backed by a container instance.|false|string||
|resource|Resource used for this container.|false|Resource||
|artifact|Artifact used for this container.|false|Artifact||
|privileged_container|Container running in privileged mode or not.|false|boolean||
@@ -322,18 +331,15 @@ a service resource has the following attributes.
|----|----|----|----|----|
|name|A unique service name. If Registry DNS is enabled, the max length is 63 characters.|true|string||
|id|A unique service id.|false|string||
-|artifact|Artifact of single-component service.|false|Artifact||
-|resource|Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.|false|Resource||
-|launch_command|The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.|false|string||
+|artifact|The default artifact for all components of the service except the components which has Artifact type set to SERVICE (optional).|false|Artifact||
+|resource|The default resource for all components of the service (optional).|false|Resource||
|launch_time|The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.|false|string (date)||
-|number_of_containers|Number of containers for each component in the service. Each component can further override this service-level global default.|false|integer (int64)||
|number_of_running_containers|In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.|false|integer (int64)||
|lifetime|Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.|false|integer (int64)||
|placement_policy|(TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.|false|PlacementPolicy||
|components|Components of a service.|false|Component array||
|configuration|Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.|false|Configuration||
-|containers|Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.|false|Container array||
-|state|State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.|false|ServiceState||
+|state|State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state.|false|ServiceState||
|quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
|queue|The YARN queue that this service should be submitted to.|false|string||
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[52/58] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd
and merge service cmd into application cmd. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba7ed7b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba7ed7b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba7ed7b6
Branch: refs/heads/yarn-native-services
Commit: ba7ed7b66a48c61086ea837f39ff0f22ffe31eca
Parents: 8ec9e72
Author: Jian He <ji...@apache.org>
Authored: Sat Oct 28 23:04:19 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:19 2017 -0800
----------------------------------------------------------------------
NOTICE.txt | 4 -
hadoop-project/pom.xml | 6 -
hadoop-yarn-project/hadoop-yarn/bin/yarn | 27 +-
.../hadoop-yarn/conf/yarn-env.sh | 18 +-
.../hadoop-yarn-services-core/pom.xml | 5 -
.../hadoop/yarn/service/ServiceMaster.java | 21 +-
.../hadoop/yarn/service/client/ServiceCLI.java | 113 -------
.../yarn/service/client/ServiceClient.java | 187 +++++------
.../client/params/AbstractActionArgs.java | 139 --------
.../client/params/AbstractArgsDelegate.java | 28 --
.../AbstractClusterBuildingActionArgs.java | 46 ---
.../service/client/params/ActionBuildArgs.java | 39 ---
.../service/client/params/ActionClientArgs.java | 71 -----
.../service/client/params/ActionCreateArgs.java | 46 ---
.../client/params/ActionDependencyArgs.java | 65 ----
.../client/params/ActionDestroyArgs.java | 37 ---
.../service/client/params/ActionExistsArgs.java | 49 ---
.../service/client/params/ActionFlexArgs.java | 50 ---
.../service/client/params/ActionFreezeArgs.java | 56 ----
.../service/client/params/ActionHelpArgs.java | 44 ---
.../service/client/params/ActionKeytabArgs.java | 76 -----
.../service/client/params/ActionListArgs.java | 76 -----
.../client/params/ActionRegistryArgs.java | 218 -------------
.../client/params/ActionResolveArgs.java | 153 ---------
.../client/params/ActionResourceArgs.java | 70 ----
.../service/client/params/ActionStatusArgs.java | 51 ---
.../service/client/params/ActionThawArgs.java | 67 ----
.../service/client/params/ActionTokensArgs.java | 78 -----
.../service/client/params/ActionUpdateArgs.java | 37 ---
.../yarn/service/client/params/ArgOps.java | 156 ---------
.../yarn/service/client/params/Arguments.java | 103 ------
.../yarn/service/client/params/ClientArgs.java | 193 -----------
.../yarn/service/client/params/CommonArgs.java | 245 --------------
.../client/params/ComponentArgsDelegate.java | 52 ---
.../client/params/DontSplitArguments.java | 34 --
.../client/params/LaunchArgsAccessor.java | 30 --
.../client/params/LaunchArgsDelegate.java | 51 ---
.../client/params/OptionArgsDelegate.java | 66 ----
.../client/params/PathArgumentConverter.java | 34 --
.../service/client/params/ServiceAMArgs.java | 50 ---
.../client/params/ServiceAMCreateAction.java | 37 ---
.../service/client/params/SliderActions.java | 79 -----
.../service/client/params/WaitArgsDelegate.java | 42 ---
.../service/client/params/WaitTimeAccessor.java | 24 --
.../hadoop/yarn/service/utils/SliderUtils.java | 6 +-
.../client/TestBuildExternalComponents.java | 19 +-
.../yarn/service/client/TestServiceCLI.java | 125 +++-----
.../hadoop/yarn/client/api/AppAdminClient.java | 222 +++++++++++++
.../hadoop/yarn/client/cli/ApplicationCLI.java | 318 ++++++++++++++++---
.../hadoop/yarn/client/cli/TestYarnCLI.java | 158 ++++++---
.../src/site/markdown/YarnCommands.md | 86 ++---
.../src/site/markdown/yarn-service/Examples.md | 12 +-
.../site/markdown/yarn-service/QuickStart.md | 46 ++-
.../markdown/yarn-service/YarnServiceAPI.md | 20 +-
54 files changed, 840 insertions(+), 3245 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index f3af2f7..bc7a26f 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -582,10 +582,6 @@ Ehcache 3.3.1,
which has the following notices:
* Ehcache V3 Copyright 2014-2016 Terracotta, Inc.
-JCommander (https://github.com/cbeust/jcommander),
-which has the following notices:
- * Copyright 2010 Cedric Beust cedric@beust.com
-
The binary distribution of this product bundles binaries of
snakeyaml (https://bitbucket.org/asomov/snakeyaml),
which has the following notices:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index d18c9a8..f99e413 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -146,7 +146,6 @@
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>1.16</snakeyaml.version>
- <jcommander.version>1.48</jcommander.version>
</properties>
<dependencyManagement>
@@ -1353,11 +1352,6 @@
<artifactId>snakeyaml</artifactId>
<version>${snakeyaml.version}</version>
</dependency>
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- <version>${jcommander.version}</version>
- </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index cc269d8..00596c2 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -31,8 +31,7 @@ function hadoop_usage
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
- hadoop_add_subcommand "apiserver" daemon "run the api-server for deploying/managing services on YARN"
- hadoop_add_subcommand "application" client "prints application(s) report/kill application"
+ hadoop_add_subcommand "app|application" client "prints application(s) report/kill application/manage long running application"
hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report"
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
hadoop_add_subcommand "cluster" client "prints cluster information"
@@ -51,7 +50,6 @@ function hadoop_usage
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "service" client "deploy/manage a service on YARN"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
@@ -71,9 +69,10 @@ function yarncmd_case
shift
case ${subcmd} in
- apiserver)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp'
+ app|application|applicationattempt|container)
+ HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
+ set -- "${subcmd}" "$@"
+ HADOOP_SUBCMD_ARGS=("$@")
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
@@ -83,11 +82,6 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_translate_cygwin_path sld
hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
;;
- application|applicationattempt|container)
- HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
- set -- "${subcmd}" "$@"
- HADOOP_SUBCMD_ARGS=("$@")
- ;;
classpath)
hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
;;
@@ -169,17 +163,6 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
- service)
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI'
- local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_HDFS_HOME}/${HDFS_DIR},\
-${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
-${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
-${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
- hadoop_translate_cygwin_path sld
- hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
- ;;
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 4bd1d3e..d865023 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -150,18 +150,6 @@
#export YARN_ROUTER_OPTS=
###
-# Services API specific parameters
-###
-# Specify the JVM options to be used when starting the services API.
-#
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# See ResourceManager for some examples
-#
-#export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')"
-
-###
# Registry DNS specific parameters
###
# For privileged registry DNS, user to run as after dropping privileges
@@ -172,3 +160,9 @@
# By default, Hadoop uses jsvc which needs to know to launch a
# server jvm.
# export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"
+
+###
+# YARN Services parameters
+###
+# Directory containing service examples
+# export YARN_SERVICE_EXAMPLES_DIR = $HADOOP_YARN_HOME/share/hadoop/yarn/yarn-service-examples
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index 851f73b..6c9a875 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -102,11 +102,6 @@
</build>
<dependencies>
<dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index 2abdae1..d58e32b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.service;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -34,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.client.params.ServiceAMArgs;
import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
@@ -51,7 +52,9 @@ public class ServiceMaster extends CompositeService {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceMaster.class);
- private static ServiceAMArgs amArgs;
+ public static final String YARNFILE_OPTION = "yarnfile";
+
+ private static String serviceDefPath;
protected ServiceContext context;
public ServiceMaster(String name) {
@@ -108,7 +111,7 @@ public class ServiceMaster extends CompositeService {
}
protected Path getAppDir() {
- return new Path(amArgs.getServiceDefPath()).getParent();
+ return new Path(serviceDefPath).getParent();
}
protected ServiceScheduler createServiceScheduler(ServiceContext context)
@@ -119,7 +122,7 @@ public class ServiceMaster extends CompositeService {
protected void loadApplicationJson(ServiceContext context,
SliderFileSystem fs) throws IOException {
context.service = ServiceApiUtil
- .loadServiceFrom(fs, new Path(amArgs.getServiceDefPath()));
+ .loadServiceFrom(fs, new Path(serviceDefPath));
LOG.info(context.service.toString());
}
@@ -138,14 +141,18 @@ public class ServiceMaster extends CompositeService {
public static void main(String[] args) throws Exception {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
- amArgs = new ServiceAMArgs(args);
- amArgs.parse();
try {
ServiceMaster serviceMaster = new ServiceMaster("Service Master");
ShutdownHookManager.get()
.addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30);
YarnConfiguration conf = new YarnConfiguration();
- new GenericOptionsParser(conf, args);
+ Options opts = new Options();
+ opts.addOption(YARNFILE_OPTION, true, "HDFS path to JSON service " +
+ "specification");
+ opts.getOption(YARNFILE_OPTION).setRequired(true);
+ GenericOptionsParser parser = new GenericOptionsParser(conf, opts, args);
+ CommandLine cmdLine = parser.getCommandLine();
+ serviceMaster.serviceDefPath = cmdLine.getOptionValue(YARNFILE_OPTION);
serviceMaster.init(conf);
serviceMaster.start();
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
deleted file mode 100644
index 928c06f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import com.beust.jcommander.ParameterException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.*;
-
-public class ServiceCLI {
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceClient.class);
- protected ServiceClient client;
-
- int exec(ClientArgs args) throws Throwable {
- if (StringUtils.isEmpty(args.getAction())) {
- System.out.println(args.usage());
- return -1;
- }
- switch (args.getAction()) {
- case ACTION_BUILD: // Upload app json onto hdfs
- client.actionBuild(args.getActionBuildArgs());
- break;
- case ACTION_START: // start the app with the pre-uploaded app json on hdfs
- client.actionStart(args.getClusterName());
- break;
- case ACTION_CREATE: // create == build + start
- client.actionCreate(args.getActionCreateArgs());
- break;
- case ACTION_STATUS:
- Service app = client.getStatus(args.getClusterName());
- System.out.println(app);
- break;
- case ACTION_FLEX:
- try {
- client.actionFlexByCLI(args);
- } catch (FileNotFoundException e) {
- System.err.println(
- args.getClusterName() + " doesn't exist: " + e.getMessage());
- return -1;
- }
- break;
- case ACTION_STOP:
- client.actionStop(args.getClusterName(), false);
- break;
- case ACTION_DESTROY: // Destroy can happen only if app is already stopped
- client.actionDestroy(args.getClusterName());
- break;
- case ACTION_DEPENDENCY: // upload dependency jars
- client.actionDependency(args.getActionDependencyArgs());
- break;
- case ACTION_UPDATE:
- client.updateLifetime(args.getClusterName(),
- args.getActionUpdateArgs().lifetime);
- break;
- case ACTION_HELP:
- LOG.info(args.usage());
- break;
- default:
- LOG.info("NOT IMPLEMENTED: " + args.getAction());
- LOG.info(args.usage());
- return -1;
- }
- return 0;
- }
-
- public ServiceCLI() {
- createServiceClient();
- }
-
- protected void createServiceClient() {
- client = new ServiceClient();
- client.init(new YarnConfiguration());
- client.start();
- }
-
- public static void main(String[] args) throws Throwable {
- ClientArgs clientArgs = new ClientArgs(args);
- try {
- clientArgs.parse();
- } catch (ParameterException | SliderException e) {
- System.err.println(e.getMessage());
- System.exit(-1);
- }
- ServiceCLI cli = new ServiceCLI();
- int res = cli.exec(clientArgs);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index f6d35e4..636b127 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
@@ -50,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -65,13 +65,6 @@ import org.apache.hadoop.yarn.service.ServiceMaster;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
-import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.client.params.CommonArgs;
import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
@@ -80,7 +73,6 @@ import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
@@ -107,16 +99,14 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public class ServiceClient extends CompositeService
- implements SliderExitCodes, YarnServiceConstants {
+public class ServiceClient extends AppAdminClient implements SliderExitCodes,
+ YarnServiceConstants {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceClient.class);
private SliderFileSystem fs;
@@ -136,10 +126,6 @@ public class ServiceClient extends CompositeService
private static EnumSet<YarnApplicationState> preRunningStates =
EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED);
- public ServiceClient() {
- super(ServiceClient.class.getName());
- }
-
@Override protected void serviceInit(Configuration configuration)
throws Exception {
fs = new SliderFileSystem(configuration);
@@ -157,28 +143,60 @@ public class ServiceClient extends CompositeService
super.serviceStop();
}
- private Service loadAppJsonFromLocalFS(
- AbstractClusterBuildingActionArgs args) throws IOException {
- File file = args.getFile();
+ public Service loadAppJsonFromLocalFS(String fileName, String serviceName,
+ Long lifetime, String queue) throws IOException, YarnException {
+ File file = new File(fileName);
+ if (!file.exists() && fileName.equals(file.getName())) {
+ String examplesDirStr = System.getenv("YARN_SERVICE_EXAMPLES_DIR");
+ String[] examplesDirs;
+ if (examplesDirStr == null) {
+ String yarnHome = System
+ .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
+ examplesDirs = new String[]{
+ yarnHome + "/share/hadoop/yarn/yarn-service-examples",
+ yarnHome + "/yarn-service-examples"
+ };
+ } else {
+ examplesDirs = StringUtils.split(examplesDirStr, ":");
+ }
+ for (String dir : examplesDirs) {
+ file = new File(MessageFormat.format("{0}/{1}/{2}.json",
+ dir, fileName, fileName));
+ if (file.exists()) {
+ break;
+ }
+ // Then look for secondary location.
+ file = new File(MessageFormat.format("{0}/{1}.json",
+ dir, fileName));
+ if (file.exists()) {
+ break;
+ }
+ }
+ }
+ if (!file.exists()) {
+ throw new YarnException("File or example could not be found: " +
+ fileName);
+ }
Path filePath = new Path(file.getAbsolutePath());
LOG.info("Loading service definition from: " + filePath);
Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
- if (args.lifetime > 0) {
- service.setLifetime(args.lifetime);
+ if (!StringUtils.isEmpty(serviceName)) {
+ service.setName(serviceName);
}
- if (!StringUtils.isEmpty(args.getServiceName())) {
- service.setName(args.getServiceName());
+ if (lifetime != null && lifetime > 0) {
+ service.setLifetime(lifetime);
}
- if (!StringUtils.isEmpty(args.queue)) {
- service.setQueue(args.queue);
+ if (!StringUtils.isEmpty(queue)) {
+ service.setQueue(queue);
}
return service;
}
- public int actionBuild(AbstractClusterBuildingActionArgs args)
- throws IOException, YarnException {
- return actionBuild(loadAppJsonFromLocalFS(args));
+ public int actionSave(String fileName, String serviceName, Long lifetime,
+ String queue) throws IOException, YarnException {
+ return actionBuild(loadAppJsonFromLocalFS(fileName, serviceName,
+ lifetime, queue));
}
public int actionBuild(Service service)
@@ -189,43 +207,13 @@ public class ServiceClient extends CompositeService
return EXIT_SUCCESS;
}
- public int actionCreate(ActionCreateArgs args)
- throws IOException, YarnException {
- Service serviceDef;
- if (args.file != null) {
- serviceDef = loadAppJsonFromLocalFS(args);
- } else if (!StringUtils.isEmpty(args.example)) {
- // create an example service
- args.file = findExampleService(args);
- serviceDef = loadAppJsonFromLocalFS(args);
- } else {
- throw new YarnException("No service definition provided!");
- }
- actionCreate(serviceDef);
+ public int actionLaunch(String fileName, String serviceName, Long lifetime,
+ String queue) throws IOException, YarnException {
+ actionCreate(loadAppJsonFromLocalFS(fileName, serviceName, lifetime,
+ queue));
return EXIT_SUCCESS;
}
- private File findExampleService(ActionCreateArgs args) throws YarnException {
- String yarnHome = System
- .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
- // First look for the standard location.
- File file = new File(MessageFormat
- .format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
- yarnHome, args.example, args.example));
- if (file.exists()) {
- return file;
- }
- // Then look for secondary location.
- file = new File(MessageFormat
- .format("{0}/yarn-service-examples/{1}/{2}.json", yarnHome,
- args.example, args.example));
- if (file.exists()) {
- return file;
- }
- throw new YarnException(
- "Example service " + args.example + " does not exist!");
- }
-
public ApplicationId actionCreate(Service service)
throws IOException, YarnException {
String serviceName = service.getName();
@@ -244,14 +232,12 @@ public class ServiceClient extends CompositeService
return appId;
}
- // Called by ServiceCLI
- protected int actionFlexByCLI(ClientArgs args)
- throws YarnException, IOException {
- ActionFlexArgs flexArgs = args.getActionFlexArgs();
+ public int actionFlex(String serviceName, Map<String, String>
+ componentCountStrings) throws YarnException, IOException {
Map<String, Long> componentCounts =
- new HashMap<>(flexArgs.getComponentMap().size());
+ new HashMap<>(componentCountStrings.size());
Service persistedService =
- ServiceApiUtil.loadService(fs, flexArgs.getServiceName());
+ ServiceApiUtil.loadService(fs, serviceName);
if (!StringUtils.isEmpty(persistedService.getId())) {
cachedAppIds.put(persistedService.getName(),
ApplicationId.fromString(persistedService.getId()));
@@ -260,8 +246,7 @@ public class ServiceClient extends CompositeService
+ " appId is null, may be not submitted to YARN yet");
}
- for (Map.Entry<String, String> entry : flexArgs.getComponentMap()
- .entrySet()) {
+ for (Map.Entry<String, String> entry : componentCountStrings.entrySet()) {
String compName = entry.getKey();
ServiceApiUtil.validateNameFormat(compName, getConfig());
Component component = persistedService.getComponent(compName);
@@ -272,11 +257,7 @@ public class ServiceClient extends CompositeService
parseNumberOfContainers(component, entry.getValue());
componentCounts.put(compName, numberOfContainers);
}
- // throw usage exception if no changes proposed
- if (componentCounts.size() == 0) {
- actionHelp(ACTION_FLEX, args);
- }
- flexComponents(args.getClusterName(), componentCounts, persistedService);
+ flexComponents(serviceName, componentCounts, persistedService);
return EXIT_SUCCESS;
}
@@ -372,6 +353,11 @@ public class ServiceClient extends CompositeService
return original;
}
+ public int actionStop(String serviceName)
+ throws YarnException, IOException {
+ return actionStop(serviceName, true);
+ }
+
public int actionStop(String serviceName, boolean waitForAppStopped)
throws YarnException, IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
@@ -442,7 +428,8 @@ public class ServiceClient extends CompositeService
return EXIT_SUCCESS;
}
- public int actionDestroy(String serviceName) throws Exception {
+ public int actionDestroy(String serviceName) throws YarnException,
+ IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
verifyNoLiveAppInRM(serviceName, "destroy");
@@ -461,7 +448,11 @@ public class ServiceClient extends CompositeService
throw new YarnException(message);
}
}
- deleteZKNode(serviceName);
+ try {
+ deleteZKNode(serviceName);
+ } catch (Exception e) {
+ throw new IOException("Could not delete zk node for " + serviceName, e);
+ }
String registryPath = ServiceRegistryUtils.registryPathForInstance(serviceName);
try {
getRegistryClient().delete(registryPath, true);
@@ -517,11 +508,6 @@ public class ServiceClient extends CompositeService
return curatorClient;
}
- private int actionHelp(String actionName, CommonArgs args)
- throws YarnException, IOException {
- throw new UsageException(CommonArgs.usage(args, actionName));
- }
-
private void verifyNoLiveAppInRM(String serviceName, String action)
throws IOException, YarnException {
Set<String> types = new HashSet<>(1);
@@ -634,9 +620,9 @@ public class ServiceClient extends CompositeService
CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
}
CLI.add(ServiceMaster.class.getCanonicalName());
- CLI.add(ACTION_CREATE, serviceName);
//TODO debugAM CLI.add(Arguments.ARG_DEBUG)
- CLI.add(Arguments.ARG_SERVICE_DEF_PATH, new Path(appRootDir, serviceName + ".json"));
+ CLI.add("-" + ServiceMaster.YARNFILE_OPTION, new Path(appRootDir,
+ serviceName + ".json"));
// pass the registry binding
CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
@@ -841,6 +827,24 @@ public class ServiceClient extends CompositeService
return ServiceState.ACCEPTED;
}
+ public String getStatusString(String appId)
+ throws IOException, YarnException {
+ ApplicationReport appReport =
+ yarnClient.getApplicationReport(ApplicationId.fromString(appId));
+
+ if (appReport.getYarnApplicationState() != RUNNING) {
+ return "";
+ }
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ return "";
+ }
+ ClientAMProtocol amProxy =
+ createAMProxy(appReport.getHost(), appReport.getRpcPort());
+ GetStatusResponseProto response =
+ amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
+ return response.getStatus();
+ }
+
public Service getStatus(String serviceName)
throws IOException, YarnException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
@@ -877,7 +881,11 @@ public class ServiceClient extends CompositeService
return this.yarnClient;
}
- public int actionDependency(ActionDependencyArgs args)
+ public int enableFastLaunch() throws IOException, YarnException {
+ return actionDependency(true);
+ }
+
+ public int actionDependency(boolean overwrite)
throws IOException, YarnException {
String currentUser = RegistryUtils.currentUser();
LOG.info("Running command as user {}", currentUser);
@@ -886,11 +894,10 @@ public class ServiceClient extends CompositeService
// Check if dependency has already been uploaded, in which case log
// appropriately and exit success (unless overwrite has been requested)
- if (fs.isFile(dependencyLibTarGzip) && !args.overwrite) {
+ if (fs.isFile(dependencyLibTarGzip) && !overwrite) {
System.out.println(String.format(
- "Dependency libs are already uploaded to %s. Use %s "
- + "if you want to re-upload", dependencyLibTarGzip.toUri(),
- Arguments.ARG_OVERWRITE));
+ "Dependency libs are already uploaded to %s.", dependencyLibTarGzip
+ .toUri()));
return EXIT_SUCCESS;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
deleted file mode 100644
index 9b7e2a4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Base args for all actions
- */
-public abstract class AbstractActionArgs extends ArgOps implements Arguments {
- protected static final Logger log =
- LoggerFactory.getLogger(AbstractActionArgs.class);
-
-
- protected AbstractActionArgs() {
- }
-
- /**
- * This is the default parameter
- */
- @Parameter
- public List<String> parameters = new ArrayList<>();
-
- /**
- * get the name: relies on arg 1 being the cluster name in all operations
- * @return the name argument, null if there is none
- */
- public String getServiceName() {
- return (parameters.isEmpty()) ? null : parameters.get(0);
- }
-
- /**
- -D name=value
-
- Define an configuration option which overrides any options in
- the configuration XML files of the image or in the image configuration
- directory. The values will be persisted.
- Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
-
- */
-
- @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions", hidden = true)
- public List<String> definitions = new ArrayList<>();
-
- /**
- * System properties
- */
- @Parameter(names = {ARG_SYSPROP}, arity = 1,
- description = "system properties in the form name value" +
- " These are set after the JVM is started.",
- hidden = true)
- public List<String> sysprops = new ArrayList<>(0);
-
-
- @Parameter(names = ARG_DEBUG, description = "Debug mode", hidden = true)
- public boolean debug = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 1;
- }
-
- /**
- * Get the name of the action
- * @return the action name
- */
- public abstract String getActionName() ;
-
- /**
- * Get the max #of params expected
- * @return the number of params in the {@link #parameters} field;
- */
- public int getMaxParams() {
- return getMinParams();
- }
-
- public void validate() throws BadCommandArgumentsException, UsageException {
-
- int minArgs = getMinParams();
- int actionArgSize = parameters.size();
- if (minArgs > actionArgSize) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
- ", Expected minimum " + minArgs + " but got " + actionArgSize);
- }
- int maxArgs = getMaxParams();
- if (maxArgs == -1) {
- maxArgs = minArgs;
- }
- if (actionArgSize > maxArgs) {
- String message = String.format("%s for action %s: limit is %d but saw %d: ",
- ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
- getActionName(), maxArgs,
- actionArgSize);
-
- log.error(message);
- int index = 1;
- StringBuilder buf = new StringBuilder(message);
- for (String actionArg : parameters) {
- log.error("[{}] \"{}\"", index++, actionArg);
- buf.append(" \"").append(actionArg).append("\" ");
- }
- throw new BadCommandArgumentsException(buf.toString());
- }
- }
-
- @Override
- public String toString() {
- return super.toString() + ": " + getActionName();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
deleted file mode 100644
index 457e357..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.yarn.service.client.params.ArgOps;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-
-/**
- * Base class for all the delegates
- */
-public class AbstractArgsDelegate extends ArgOps implements Arguments {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
deleted file mode 100644
index 4ecbe9c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-import java.io.File;
-
-/**
- * Abstract Action to build things; shares args across build and
- * list
- */
-public abstract class AbstractClusterBuildingActionArgs
- extends AbstractActionArgs {
- @Parameter(names = { ARG_FILE, ARG_FILE_SHORT },
- description = "The path to the service definition file in JSON format.")
- public File file;
-
- public File getFile() {
- return file;
- }
-
- @Parameter(names = {
- ARG_QUEUE, ARG_SHORT_QUEUE}, description = "Queue to submit the service")
- public String queue;
-
- @Parameter(names = {
- ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
- public long lifetime;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
deleted file mode 100644
index 0228125..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-@Parameters(commandNames = { SliderActions.ACTION_BUILD},
- commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
-
-public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_BUILD;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException {
- if (file == null) {
- throw new BadCommandArgumentsException("No service definition provided.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
deleted file mode 100644
index c43d61a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_CLIENT},
- commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
-
-public class ActionClientArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CLIENT;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install client")
- public boolean install;
-
- @Parameter(names = {ARG_NAME},
- description = "The name of the service")
- public String name;
-
- @Parameter(names = {ARG_PACKAGE},
- description = "Path to app package")
- public String packageURI;
-
- @Parameter(names = {ARG_DEST},
- description = "The location where to install the client")
- public File installLocation;
-
- @Parameter(names = {ARG_CONFIG},
- description = "Client configuration")
- public File clientConfig;
-
- /**
- * Get the min #of params expected
- *
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
deleted file mode 100644
index 039f528..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
-
- @Parameter(names = { ARG_EXAMPLE, ARG_EXAMPLE_SHORT },
- description = "The name of the example service such as sleeper")
- public String example;
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException {
- if (file == null && example == null) {
- throw new BadCommandArgumentsException("No service definition provided.");
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
deleted file mode 100644
index 699e6ab..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY },
- commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY)
-public class ActionDependencyArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_DEPENDENCY;
- }
-
- @Parameter(names = { ARG_UPLOAD }, required = true,
- description = "Upload AM libraries to HDFS for this client version")
- public boolean upload;
-
- @Parameter(names = { ARG_OVERWRITE },
- description = "Overwrite current uploaded dependency libs")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- *
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
-
- if (!upload) {
- throw new UsageException("Option " + ARG_UPLOAD + " is mandatory");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
deleted file mode 100644
index 8c41c04..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_DESTROY},
- commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
-
-public class ActionDestroyArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_DESTROY;
- }
-
- @Parameter(names = {ARG_FORCE},
- description = "force the operation")
- public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
deleted file mode 100644
index 088ad47..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_EXISTS},
- commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
-
-public class ActionExistsArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_EXISTS;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "verify that the service is running")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "verify that the service is in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any service report")
- public File out;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
deleted file mode 100644
index b7acf58..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.List;
-import java.util.Map;
-
-@Parameters(commandNames = { SliderActions.ACTION_FLEX},
- commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
-
-public class ActionFlexArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_FLEX;
- }
-
- @ParametersDelegate
- public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
-
- /**
- * Get the component mapping (may be empty, but never null)
- * @return mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getComponentMap() throws
- BadCommandArgumentsException {
- return componentDelegate.getComponentMap();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
deleted file mode 100644
index aecf0eb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-@Parameters(commandNames = { SliderActions.ACTION_STOP },
- commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
-
-public class ActionFreezeArgs extends AbstractActionArgs implements
- WaitTimeAccessor {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STOP;
- }
-
- public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
- @ParametersDelegate
- public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
-
- @Override
- public int getWaittime() {
- return waitDelegate.getWaittime();
- }
-
- @Override
- public void setWaittime(int waittime) {
- waitDelegate.setWaittime(waittime);
- }
-
- @Parameter(names={ARG_MESSAGE},
- description = "reason for the operation")
- public String message = FREEZE_COMMAND_ISSUED;
-
- @Parameter(names = {ARG_FORCE},
- description = "force the operation")
- public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
deleted file mode 100644
index 51aa88a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-/**
- * The Help command
- */
-@Parameters(commandNames = { SliderActions.ACTION_HELP},
- commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
-public class ActionHelpArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_HELP;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
deleted file mode 100644
index 7e51457..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_KEYTAB},
- commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
-
-public class ActionKeytabArgs extends AbstractActionArgs {
-
- public ActionKeytabArgs() {
- super();
- }
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_INSTALL_KEYTAB;
- }
-
- @Parameter(names = {ARG_KEYTABINSTALL},
- description = "Install the keytab")
- public boolean install;
-
- @Parameter(names = {ARG_KEYTABDELETE},
- description = "Delete the keytab")
- public boolean delete;
-
- @Parameter(names = {ARG_KEYTABLIST},
- description = "List of installed keytabs")
- public boolean list;
-
- @Parameter(names = {ARG_KEYTAB},
- description = "Path or name of the keytab")
- public String keytab;
-
- @Parameter(names = {ARG_FOLDER},
- description = "The name of the folder in which to store the keytab")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
deleted file mode 100644
index c05e602..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_LIST},
- commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
-public class ActionListArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_LIST;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "List only live service instances")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "list only applications in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_VERBOSE},
- description = "print out information in details")
- public boolean verbose = false;
-
- @Parameter(names = {ARG_CONTAINERS},
- description = "List containers of a service instance")
- public boolean containers;
-
- @Parameter(names = {ARG_VERSION},
- description = "Filter containers by app version (used with " +
- ARG_CONTAINERS + ")")
- public String version;
-
- @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
- description = "Filter containers by component names (used with " +
- ARG_CONTAINERS + ")")
- public Set<String> components = new HashSet<>(0);
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
deleted file mode 100644
index 3e53418..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-import java.io.File;
-
-/**
- * Registry actions
- *
- * --instance {app name}, if a / is in it, refers underneath?
- * --dest {destfile}
- * --list : list instances of slider service
- * --listfiles
- */
-@Parameters(commandNames = {ACTION_REGISTRY},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-
-public class ActionRegistryArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_REGISTRY
- + " ("
- + Arguments.ARG_LIST + "|"
- + Arguments.ARG_LISTCONF + "|"
- + Arguments.ARG_LISTEXP + "|"
- + Arguments.ARG_LISTFILES + "|"
- + Arguments.ARG_GETCONF + "|"
- + Arguments.ARG_GETEXP + "> "
- + Arguments.ARG_NAME + " <name> "
- + " )"
- + "[" + Arguments.ARG_VERBOSE + "] "
- + "[" + Arguments.ARG_USER + "] "
- + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
- + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
- + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
- + System.getProperty("line.separator")
- + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
- ;
- public ActionRegistryArgs() {
- }
-
- public ActionRegistryArgs(String name) {
- this.name = name;
- }
-
- @Override
- public String getActionName() {
- return ACTION_REGISTRY;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_LISTCONF},
- description = "list configurations")
- public boolean listConf;
-
- @Parameter(names = {ARG_GETCONF},
- description = "get configuration")
- public String getConf;
-
- @Parameter(names = {ARG_LISTEXP},
- description = "list exports")
- public boolean listExports;
-
- @Parameter(names = {ARG_GETEXP},
- description = "get export")
- public String getExport;
-
- @Parameter(names = {ARG_LISTFILES},
- description = "list files")
- public String listFiles;
-
- @Parameter(names = {ARG_GETFILES},
- description = "get files")
- public String getFiles;
-
- //--format
- @Parameter(names = ARG_FORMAT,
- description = "Format for a response: <xml|json|properties>")
- public String format = ConfigFormat.XML.toString() ;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
- description = "Output destination")
- public File out;
-
- @Parameter(names = {ARG_NAME},
- description = "name of an instance")
- public String name;
-
- @Parameter(names = {ARG_SERVICETYPE},
- description = "optional service type")
- public String serviceType = YarnServiceConstants.APP_TYPE;
-
- @Parameter(names = {ARG_VERBOSE},
- description = "verbose output")
- public boolean verbose;
-
- @Parameter(names = {ARG_INTERNAL},
- description = "fetch internal registry entries")
- public boolean internal;
-
- @Parameter(names = {ARG_USER},
- description = "the name of the user whose service is being resolved")
- public String user;
-
- /**
- * validate health of all the different operations
- * @throws BadCommandArgumentsException
- */
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
-
- //verify that at most one of the operations is set
- int gets = s(getConf) + s(getFiles) + s(getExport);
- int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
- int set = lists + gets;
- if (set > 1) {
- throw new UsageException(USAGE);
- }
-
- if (out != null && ( set == 0)) {
- throw new UsageException("output path"
- + " is only supported on 'get' operations: ");
- }
- if (!list && !is(name)) {
- throw new UsageException("Argument " + ARG_NAME
- +" missing: ");
-
- }
- }
-
- private int s(String arg) {
- return is(arg) ? 1 : 0;
- }
-
- private boolean is(String arg) {
- return arg != null;
- }
-
- private int s(boolean arg) {
- return arg ? 1 : 0;
- }
-
- private String ifdef(String arg, boolean val) {
- return val ? (arg + " "): "";
- }
-
- private String ifdef(String arg, String val) {
- if (is(val)) {
- return arg + " " + val + " ";
- } else {
- return "";
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_REGISTRY);
- sb.append(' ');
- sb.append(ifdef(ARG_LIST, list));
- sb.append(ifdef(ARG_LISTCONF, listConf));
- sb.append(ifdef(ARG_LISTFILES, listFiles));
- sb.append(ifdef(ARG_GETCONF, getConf));
- sb.append(ifdef(ARG_GETFILES, getFiles));
-
- sb.append(ifdef(ARG_NAME, name));
- sb.append(ifdef(ARG_SERVICETYPE, serviceType));
-
-
- sb.append(ifdef(ARG_VERBOSE, verbose));
- sb.append(ifdef(ARG_INTERNAL, internal));
-
- if (out != null) {
- sb.append(ifdef(ARG_OUTPUT, out.toString()));
- }
- sb.append(ifdef(ARG_FORMAT, format));
-
- return sb.toString();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/58] [abbrv] hadoop git commit: YARN-6626. Embed REST API service
into RM. Contributed by Eric Yang
Posted by ji...@apache.org.
YARN-6626. Embed REST API service into RM. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7230215
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7230215
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7230215
Branch: refs/heads/yarn-native-services
Commit: c72302157907b44b71bc15008aabd9cf3d806f12
Parents: 673c0db
Author: Jian He <ji...@apache.org>
Authored: Thu Sep 28 16:29:22 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:17 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 2 ++
.../hadoop/yarn/service/webapp/ApiServer.java | 11 +++++++
.../hadoop-yarn-services-core/pom.xml | 6 ----
.../yarn/service/api/records/Artifact.java | 8 +++++
.../yarn/service/api/records/ConfigFile.java | 8 +++++
.../service/api/records/ReadinessCheck.java | 8 +++++
.../src/main/resources/yarn-default.xml | 8 +++++
.../server/resourcemanager/webapp/RMWebApp.java | 19 +++++++++++
.../site/markdown/yarn-service/QuickStart.md | 34 +++++++++++++++++---
9 files changed, 93 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 640e86e..4799137 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -336,6 +336,8 @@ public class YarnConfiguration extends Configuration {
public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
+ "webapp.ui2.war-file-path";
+ public static final String YARN_API_SERVICES_ENABLE = "yarn."
+ + "webapp.api-service.enable";
public static final String RM_RESOURCE_TRACKER_ADDRESS =
RM_PREFIX + "resource-tracker.address";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index e8286ef..4fe12e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.yarn.service.webapp;
+import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.VersionInfo;
@@ -57,6 +58,16 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
@Singleton
@Path(CONTEXT_ROOT)
public class ApiServer {
+
+ public ApiServer() {
+ super();
+ }
+
+ @Inject
+ public ApiServer(Configuration conf) {
+ super();
+ }
+
private static final Logger LOG =
LoggerFactory.getLogger(ApiServer.class);
private static Configuration YARN_CONFIG = new YarnConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index fb07edc..205a64d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -245,12 +245,6 @@
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
index 4328c6f..ce062cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
@@ -23,6 +23,9 @@ import io.swagger.annotations.ApiModelProperty;
import java.io.Serializable;
import java.util.Objects;
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlType;
+
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
@@ -42,6 +45,11 @@ public class Artifact implements Serializable {
private String id = null;
+ /**
+ * Artifact Type. DOCKER, TARBALL or SERVICE
+ **/
+ @XmlType(name = "artifact_type")
+ @XmlEnum
public enum TypeEnum {
DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
index c1502c7..fb088e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -26,7 +26,10 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
@@ -45,6 +48,11 @@ import java.util.Objects;
public class ConfigFile implements Serializable {
private static final long serialVersionUID = -7009402089417704612L;
+ /**
+ * Config Type. XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported.
+ **/
+ @XmlType(name = "config_type")
+ @XmlEnum
public enum TypeEnum {
XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
"TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
index 1a25a4c..b25828f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -25,6 +25,9 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlType;
+
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -43,6 +46,11 @@ import org.apache.hadoop.classification.InterfaceStability;
public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L;
+ /**
+ * Type. HTTP and PORT
+ **/
+ @XmlType(name = "type")
+ @XmlEnum
public enum TypeEnum {
HTTP("HTTP"),
PORT("PORT");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index f4b2e61..e90d0f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -212,6 +212,14 @@
</property>
<property>
+ <description>
+ Enable services rest api on ResourceManager.
+ </description>
+ <name>yarn.webapp.api-service.enable</name>
+ <value>false</value>
+ </property>
+
+ <property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>${yarn.resourcemanager.hostname}:8031</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index 4e36665..bee9354 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -22,6 +22,8 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
import java.net.InetSocketAddress;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.RMHAUtils;
@@ -38,8 +40,12 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
*/
public class RMWebApp extends WebApp implements YarnWebParams {
+ private static final Log LOG =
+ LogFactory.getLog(RMWebApp.class.getName());
private final ResourceManager rm;
private boolean standby = false;
+ private final static String APISERVER =
+ "org.apache.hadoop.yarn.service.webapp.ApiServer";
public RMWebApp(ResourceManager rm) {
this.rm = rm;
@@ -53,6 +59,19 @@ public class RMWebApp extends WebApp implements YarnWebParams {
bind(RMWebApp.class).toInstance(this);
if (rm != null) {
+ boolean enableServiceApi = rm.getConfig()
+ .getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false);
+ if (enableServiceApi) {
+ try {
+ // Use reflection here to load ApiServer class,
+ // this is done to avoid creating cyclic dependency
+ // between maven projects.
+ Class<?> apiServer = Class.forName(APISERVER);
+ bind(apiServer);
+ } catch (ClassNotFoundException e) {
+ LOG.warn("ApiServer REST API is not activated.");
+ }
+ }
bind(ResourceManager.class).toInstance(rm);
}
route("/", RmController.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index 15df0cd..afcd5f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -101,17 +101,41 @@ yarn service destroy ${SERVICE_NAME}
```
## Manage services on YARN via REST API
-Below steps walk you through deploying services on YARN via REST API.
- Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
-### Start API-Server for deploying services on YARN
-API server is the service that sits in front of YARN ResourceManager and lets users submit their API specs via HTTP.
+
+YARN API Server REST API can be activated in two modes: embedded or standalone.
+
+### Start Embedded API-Server as part of ResourceManager
+For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
+
+```
+ <property>
+ <description>
+ Enable services rest api on ResourceManager.
+ </description>
+ <name>yarn.webapp.api-service.enable</name>
+ <value>true</value>
+ </property>
+```
+
+Services can be deployed on YARN through the ResourceManager web endpoint.
+
+### Start Standalone API-Server for deploying services on YARN
+API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP.
```
yarn --daemon start apiserver
```
The above command starts the API Server on the localhost at port 9191 by default.
+Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
+
### Deploy a service
-POST the aforementioned example service definition to the api-server endpoint:
+
+POST the aforementioned example service definition to the ResourceManager api-server endpoint:
+```
+POST http://localhost:8088/ws/v1/services
+```
+
+Or standalone API server:
```
POST http://localhost:9191/ws/v1/services
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
deleted file mode 100644
index 982a114..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.ServiceScheduler;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.Date;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-
-import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
-import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.*;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.*;
-
-public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
- Comparable<ComponentInstance> {
- private static final Logger LOG =
- LoggerFactory.getLogger(ComponentInstance.class);
-
- private StateMachine<ComponentInstanceState, ComponentInstanceEventType,
- ComponentInstanceEvent> stateMachine;
- private Component component;
- private final ReadLock readLock;
- private final WriteLock writeLock;
-
- private ComponentInstanceId compInstanceId = null;
- private Path compInstanceDir;
- private Container container;
- private YarnRegistryViewForProviders yarnRegistryOperations;
- private FileSystem fs;
- private boolean timelineServiceEnabled = false;
- private ServiceTimelinePublisher serviceTimelinePublisher;
- private ServiceScheduler scheduler;
- private BoundedAppender diagnostics = new BoundedAppender(64 * 1024);
- private volatile ScheduledFuture containerStatusFuture;
- private volatile ContainerStatus status;
- private long containerStartedTime = 0;
- // This container object is used for rest API query
- private org.apache.hadoop.yarn.service.api.records.Container containerSpec;
-
- private static final StateMachineFactory<ComponentInstance,
- ComponentInstanceState, ComponentInstanceEventType, ComponentInstanceEvent>
- stateMachineFactory =
- new StateMachineFactory<ComponentInstance, ComponentInstanceState,
- ComponentInstanceEventType, ComponentInstanceEvent>(INIT)
- .addTransition(INIT, RUNNING_BUT_UNREADY, STARTED,
- new ContainerStartedTransition())
-
- //From Running
- .addTransition(RUNNING_BUT_UNREADY, INIT, STOP,
- new ContainerStoppedTransition())
- .addTransition(RUNNING_BUT_UNREADY, READY, BECOME_READY,
- new ContainerBecomeReadyTransition())
-
- // FROM READY
- .addTransition(READY, RUNNING_BUT_UNREADY, BECOME_NOT_READY,
- new ContainerBecomeNotReadyTransition())
- .addTransition(READY, INIT, STOP, new ContainerStoppedTransition())
- .installTopology();
-
-
-
- public ComponentInstance(Component component,
- ComponentInstanceId compInstanceId) {
- this.stateMachine = stateMachineFactory.make(this);
- this.component = component;
- this.compInstanceId = compInstanceId;
- this.scheduler = component.getScheduler();
- this.yarnRegistryOperations =
- component.getScheduler().getYarnRegistryOperations();
- this.serviceTimelinePublisher =
- component.getScheduler().getServiceTimelinePublisher();
- if (YarnConfiguration
- .timelineServiceV2Enabled(component.getScheduler().getConfig())) {
- this.timelineServiceEnabled = true;
- }
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- this.readLock = lock.readLock();
- this.writeLock = lock.writeLock();
- this.fs = scheduler.getContext().fs.getFileSystem();
- }
-
- private static class ContainerStartedTransition extends BaseTransition {
- @Override public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- // Query container status for ip and host
- compInstance.containerStatusFuture =
- compInstance.scheduler.executorService.scheduleAtFixedRate(
- new ContainerStatusRetriever(compInstance.scheduler,
- compInstance.getContainerId(), compInstance), 0, 1,
- TimeUnit.SECONDS);
-
- org.apache.hadoop.yarn.service.api.records.Container container =
- new org.apache.hadoop.yarn.service.api.records.Container();
- container.setId(compInstance.getContainerId().toString());
- container.setLaunchTime(new Date());
- container.setState(ContainerState.RUNNING_BUT_UNREADY);
- container.setBareHost(compInstance.container.getNodeId().getHost());
- container.setComponentName(compInstance.getCompInstanceName());
- if (compInstance.containerSpec != null) {
- // remove the previous container.
- compInstance.getCompSpec().removeContainer(compInstance.containerSpec);
- }
- compInstance.containerSpec = container;
- compInstance.getCompSpec().addContainer(container);
- compInstance.containerStartedTime = System.currentTimeMillis();
-
- if (compInstance.timelineServiceEnabled) {
- compInstance.serviceTimelinePublisher
- .componentInstanceStarted(container, compInstance);
- }
- }
- }
-
- private static class ContainerBecomeReadyTransition extends BaseTransition {
- @Override
- public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- compInstance.component.incContainersReady();
- compInstance.containerSpec.setState(ContainerState.READY);
- }
- }
-
- private static class ContainerBecomeNotReadyTransition extends BaseTransition {
- @Override
- public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- compInstance.component.decContainersReady();
- compInstance.containerSpec.setState(ContainerState.RUNNING_BUT_UNREADY);
- }
- }
-
- private static class ContainerStoppedTransition extends BaseTransition {
- @Override
- public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- // re-ask the failed container.
- Component comp = compInstance.component;
- comp.requestContainers(1);
- LOG.info(compInstance.getCompInstanceId()
- + ": Container completed. Requested a new container." + System
- .lineSeparator() + " exitStatus={}, diagnostics={}.",
- event.getStatus().getExitStatus(),
- event.getStatus().getDiagnostics());
- String containerDiag =
- compInstance.getCompInstanceId() + ": " + event.getStatus()
- .getDiagnostics();
- compInstance.diagnostics.append(containerDiag + System.lineSeparator());
-
- boolean shouldExit = false;
- // check if it exceeds the failure threshold
- if (comp.currentContainerFailure > comp.maxContainerFailurePerComp) {
- String exitDiag = MessageFormat.format(
- "[COMPONENT {0}]: Failed {1} times, exceeded the limit - {2}. Shutting down now... "
- + System.lineSeparator(),
- comp.getName(), comp.currentContainerFailure, comp.maxContainerFailurePerComp);
- compInstance.diagnostics.append(exitDiag);
- // append to global diagnostics that will be reported to RM.
- comp.getScheduler().getDiagnostics().append(containerDiag);
- comp.getScheduler().getDiagnostics().append(exitDiag);
- LOG.warn(exitDiag);
- shouldExit = true;
- }
-
- // clean up registry
- // hdfs dir content will be overwritten when a new container gets started,
- // so no need remove.
- compInstance.scheduler.executorService
- .submit(compInstance::cleanupRegistry);
-
- // remove the failed ContainerId -> CompInstance mapping
- comp.getScheduler().removeLiveCompInstance(event.getContainerId());
-
- if (compInstance.timelineServiceEnabled) {
- // record in ATS
- compInstance.serviceTimelinePublisher
- .componentInstanceFinished(compInstance,
- event.getStatus().getExitStatus(), event.getStatus().getState(),
- containerDiag);
- }
-
- compInstance.containerSpec.setState(ContainerState.STOPPED);
- if (shouldExit) {
- // Sleep for 5 seconds in hope that the state can be recorded in ATS.
- // in case there's a client polling the comp state, it can be notified.
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- LOG.error("Interrupted on sleep while exiting.", e);
- }
- ExitUtil.terminate(-1);
- }
- }
- }
-
- public ComponentInstanceState getState() {
- this.readLock.lock();
-
- try {
- return this.stateMachine.getCurrentState();
- } finally {
- this.readLock.unlock();
- }
- }
-
- @Override
- public void handle(ComponentInstanceEvent event) {
- try {
- writeLock.lock();
- ComponentInstanceState oldState = getState();
- try {
- stateMachine.doTransition(event.getType(), event);
- } catch (InvalidStateTransitionException e) {
- LOG.error(getCompInstanceId() + ": Invalid event " + event.getType() +
- " at " + oldState, e);
- }
- if (oldState != getState()) {
- LOG.info(getCompInstanceId() + " Transitioned from " + oldState + " to "
- + getState() + " on " + event.getType() + " event");
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- public void setContainer(Container container) {
- this.container = container;
- this.compInstanceId.setContainerId(container.getId());
- }
-
- public String getCompInstanceName() {
- return compInstanceId.getCompInstanceName();
- }
-
- public ContainerStatus getContainerStatus() {
- return status;
- }
-
- public void updateContainerStatus(ContainerStatus status) {
- this.status = status;
- org.apache.hadoop.yarn.service.api.records.Container container =
- getCompSpec().getContainer(getContainerId().toString());
- if (container != null) {
- container.setIp(StringUtils.join(",", status.getIPs()));
- container.setHostname(status.getHost());
- if (timelineServiceEnabled) {
- serviceTimelinePublisher.componentInstanceUpdated(container);
- }
- }
- updateServiceRecord(yarnRegistryOperations, status);
- }
-
- public ContainerId getContainerId() {
- return container.getId();
- }
-
- public String getCompName() {
- return compInstanceId.getCompName();
- }
-
- public void setCompInstanceDir(Path dir) {
- this.compInstanceDir = dir;
- }
-
- public Component getComponent() {
- return component;
- }
-
- public Container getContainer() {
- return container;
- }
-
- public ComponentInstanceId getCompInstanceId() {
- return compInstanceId;
- }
-
- public NodeId getNodeId() {
- return this.container.getNodeId();
- }
-
- public org.apache.hadoop.yarn.service.api.records.Component getCompSpec() {
- return component.getComponentSpec();
- }
-
- private static class BaseTransition implements
- SingleArcTransition<ComponentInstance, ComponentInstanceEvent> {
-
- @Override public void transition(ComponentInstance compInstance,
- ComponentInstanceEvent event) {
- }
- }
-
- public ProbeStatus ping() {
- if (component.getProbe() == null) {
- ProbeStatus status = new ProbeStatus();
- status.setSuccess(true);
- return status;
- }
- return component.getProbe().ping(this);
- }
-
- // Write service record into registry
- private void updateServiceRecord(
- YarnRegistryViewForProviders yarnRegistry, ContainerStatus status) {
- ServiceRecord record = new ServiceRecord();
- String containerId = status.getContainerId().toString();
- record.set(YarnRegistryAttributes.YARN_ID, containerId);
- record.description = getCompInstanceName();
- record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
- PersistencePolicies.CONTAINER);
- record.set("yarn:ip", status.getIPs());
- record.set("yarn:hostname", status.getHost());
- try {
- yarnRegistry
- .putComponent(RegistryPathUtils.encodeYarnID(containerId), record);
- } catch (IOException e) {
- LOG.error(
- "Failed to update service record in registry: " + containerId + "");
- }
- }
-
- // Release the container , cleanup registry, hdfs dir, and record in ATS
- public void destroy() {
- LOG.info(getCompInstanceId() + ": Flexed down by user, destroying.");
- diagnostics.append(getCompInstanceId() + ": Flexed down by user");
- if (container != null) {
- scheduler.removeLiveCompInstance(container.getId());
- component.getScheduler().getAmRMClient()
- .releaseAssignedContainer(container.getId());
- getCompSpec().removeContainer(containerSpec);
- }
- if (timelineServiceEnabled) {
- serviceTimelinePublisher
- .componentInstanceFinished(this, KILLED_BY_APPMASTER, COMPLETE,
- diagnostics.toString());
- }
- scheduler.executorService.submit(this::cleanupRegistryAndCompHdfsDir);
- }
-
- private void cleanupRegistry() {
- ContainerId containerId = getContainerId();
- String cid = RegistryPathUtils.encodeYarnID(containerId.toString());
- try {
- yarnRegistryOperations.deleteComponent(getCompInstanceId(), cid);
- } catch (IOException e) {
- LOG.error(getCompInstanceId() + ": Failed to delete registry", e);
- }
- }
-
- //TODO Maybe have a dedicated cleanup service.
- public void cleanupRegistryAndCompHdfsDir() {
- cleanupRegistry();
- try {
- if (compInstanceDir != null && fs.exists(compInstanceDir)) {
- boolean deleted = fs.delete(compInstanceDir, true);
- if (!deleted) {
- LOG.error(getCompInstanceId()
- + ": Failed to delete component instance dir: "
- + compInstanceDir);
- } else {
- LOG.info(getCompInstanceId() + ": Deleted component instance dir: "
- + compInstanceDir);
- }
- }
- } catch (IOException e) {
- LOG.warn(getCompInstanceId() + ": Failed to delete directory", e);
- }
- }
-
- // Query container status until ip and hostname are available and update
- // the service record into registry service
- private static class ContainerStatusRetriever implements Runnable {
- private ContainerId containerId;
- private NodeId nodeId;
- private NMClient nmClient;
- private ComponentInstance instance;
- ContainerStatusRetriever(ServiceScheduler scheduler,
- ContainerId containerId, ComponentInstance instance) {
- this.containerId = containerId;
- this.nodeId = instance.getNodeId();
- this.nmClient = scheduler.getNmClient().getClient();
- this.instance = instance;
- }
- @Override public void run() {
- ContainerStatus status = null;
- try {
- status = nmClient.getContainerStatus(containerId, nodeId);
- } catch (Exception e) {
- if (e instanceof YarnException) {
- throw new YarnRuntimeException(
- instance.compInstanceId + " Failed to get container status on "
- + nodeId + " , cancelling.", e);
- }
- LOG.error(instance.compInstanceId + " Failed to get container status on "
- + nodeId + ", will try again", e);
- return;
- }
- if (SliderUtils.isEmpty(status.getIPs()) || SliderUtils
- .isUnset(status.getHost())) {
- return;
- }
- instance.updateContainerStatus(status);
- LOG.info(
- instance.compInstanceId + " IP = " + status.getIPs() + ", host = "
- + status.getHost() + ", cancel container status retriever");
- instance.containerStatusFuture.cancel(false);
- }
- }
-
- @Override
- public int compareTo(ComponentInstance to) {
- long delta = containerStartedTime - to.containerStartedTime;
- if (delta == 0) {
- return getCompInstanceId().compareTo(to.getCompInstanceId());
- } else if (delta < 0) {
- return -1;
- } else {
- return 1;
- }
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- ComponentInstance instance = (ComponentInstance) o;
-
- if (containerStartedTime != instance.containerStartedTime)
- return false;
- return compInstanceId.equals(instance.compInstanceId);
- }
-
- @Override public int hashCode() {
- int result = compInstanceId.hashCode();
- result = 31 * result + (int) (containerStartedTime ^ (containerStartedTime
- >>> 32));
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEvent.java
deleted file mode 100644
index 14a9e09..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEvent.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-
-public class ComponentInstanceEvent
- extends AbstractEvent<ComponentInstanceEventType> {
-
- private ContainerId id;
- private ContainerStatus status;
- private boolean shouldDestroy = false;
-
- public ComponentInstanceEvent(ContainerId containerId,
- ComponentInstanceEventType componentInstanceEventType) {
- super(componentInstanceEventType);
- this.id = containerId;
- }
-
- public ContainerId getContainerId() {
- return id;
- }
-
- public ContainerStatus getStatus() {
- return this.status;
- }
-
- public ComponentInstanceEvent setStatus(ContainerStatus status) {
- this.status = status;
- return this;
- }
-
- public void setShouldDestroy() {
- shouldDestroy = true;
- }
-
- public boolean shouldDestroy() {
- return shouldDestroy;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEventType.java
deleted file mode 100644
index b3fe1e6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceEventType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-public enum ComponentInstanceEventType {
-
- STARTED,
- STOP,
- BECOME_READY,
- BECOME_NOT_READY
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceId.java
deleted file mode 100644
index c3c55d9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceId.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-
-public class ComponentInstanceId implements Comparable<ComponentInstanceId> {
-
- private long Id;
- private String name;
- private ContainerId containerId;
-
- public ComponentInstanceId(long id, String name) {
- Id = id;
- this.name = name;
- }
-
- public long getId() {
- return Id;
- }
-
- public String getCompName() {
- return name;
- }
-
- public String getCompInstanceName() {
- return getCompName() + "-" + getId();
- }
-
- public void setContainerId(ContainerId containerId) {
- this.containerId = containerId;
- }
-
- @Override
- public String toString() {
- if (containerId == null) {
- return "[COMPINSTANCE " + getCompInstanceName() + "]";
- } else {
- return "[COMPINSTANCE " + getCompInstanceName() + " : " + containerId + "]";
- }
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- ComponentInstanceId that = (ComponentInstanceId) o;
-
- if (getId() != that.getId())
- return false;
- return getCompName() != null ? getCompName().equals(that.getCompName()) :
- that.getCompName() == null;
-
- }
-
- @Override public int hashCode() {
- int result = (int) (getId() ^ (getId() >>> 32));
- result = 31 * result + (getCompName() != null ? getCompName().hashCode() : 0);
- return result;
- }
-
- @Override
- public int compareTo(ComponentInstanceId to) {
- int delta = this.getCompName().compareTo(to.getCompName());
- if (delta == 0) {
- return Long.compare(this.getId(), to.getId());
- } else if (delta < 0) {
- return -1;
- } else {
- return 1;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceState.java
deleted file mode 100644
index f2d8cea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstanceState.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-public enum ComponentInstanceState {
- INIT,
- RUNNING_BUT_UNREADY,
- READY,
- UPGRADING
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
deleted file mode 100644
index 331871a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.component;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
-import org.apache.hadoop.yarn.service.ContainerFailureTracker;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.ServiceScheduler;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
-import org.apache.hadoop.yarn.state.MultipleArcTransition;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.MonitorUtils;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.Probe;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
-import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STARTED;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STOP;
-import static org.apache.hadoop.yarn.service.component.ComponentState.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
-
-public class Component implements EventHandler<ComponentEvent> {
- private static final Logger LOG = LoggerFactory.getLogger(Component.class);
-
- private org.apache.hadoop.yarn.service.api.records.Component componentSpec;
- private long allocateId;
- private Priority priority;
- private ServiceMetrics componentMetrics;
- private ServiceScheduler scheduler;
- private ServiceContext context;
- private AMRMClientAsync<ContainerRequest> amrmClient;
- private AtomicLong instanceIdCounter = new AtomicLong();
- private Map<ComponentInstanceId, ComponentInstance> compInstances =
- new ConcurrentHashMap<>();
- // component instances to be assigned with a container
- private List<ComponentInstance> pendingInstances = new LinkedList<>();
- private ContainerFailureTracker failureTracker;
- private Probe probe;
- private final ReentrantReadWriteLock.ReadLock readLock;
- private final ReentrantReadWriteLock.WriteLock writeLock;
- public int maxContainerFailurePerComp;
- // The number of containers failed since last reset. This excludes preempted,
- // disk_failed containers etc. This will be reset to 0 periodically.
- public volatile int currentContainerFailure;
-
- private StateMachine<ComponentState, ComponentEventType, ComponentEvent>
- stateMachine;
- private AsyncDispatcher compInstanceDispatcher;
- private static final StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>
- stateMachineFactory =
- new StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>(
- INIT)
- // INIT will only got to FLEXING
- .addTransition(INIT, EnumSet.of(STABLE, FLEXING),
- FLEX, new FlexComponentTransition())
-
- // container allocated by RM
- .addTransition(FLEXING, FLEXING, CONTAINER_ALLOCATED,
- new ContainerAllocatedTransition())
- // container launched on NM
- .addTransition(FLEXING, EnumSet.of(STABLE, FLEXING),
- CONTAINER_STARTED, new ContainerStartedTransition())
- // container failed while flexing
- .addTransition(FLEXING, FLEXING, CONTAINER_COMPLETED,
- new ContainerCompletedTransition())
- // Flex while previous flex is still in progress
- .addTransition(FLEXING, EnumSet.of(FLEXING), FLEX,
- new FlexComponentTransition())
-
- // container failed while stable
- .addTransition(STABLE, FLEXING, CONTAINER_COMPLETED,
- new ContainerCompletedTransition())
- // Ignore surplus container
- .addTransition(STABLE, STABLE, CONTAINER_ALLOCATED,
- new ContainerAllocatedTransition())
- // Flex by user
- // For flex up, go to FLEXING state
- // For flex down, go to STABLE state
- .addTransition(STABLE, EnumSet.of(STABLE, FLEXING),
- FLEX, new FlexComponentTransition())
- .installTopology();
-
- public Component(
- org.apache.hadoop.yarn.service.api.records.Component component,
- long allocateId, ServiceContext context) {
- this.allocateId = allocateId;
- this.priority = Priority.newInstance((int) allocateId);
- this.componentSpec = component;
- componentMetrics = ServiceMetrics.register(component.getName(),
- "Metrics for component " + component.getName());
- componentMetrics
- .tag("type", "Metrics type [component or service]", "component");
- this.scheduler = context.scheduler;
- this.context = context;
- amrmClient = scheduler.getAmRMClient();
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- this.readLock = lock.readLock();
- this.writeLock = lock.writeLock();
- this.stateMachine = stateMachineFactory.make(this);
- compInstanceDispatcher = scheduler.getCompInstanceDispatcher();
- failureTracker =
- new ContainerFailureTracker(context, this);
- probe = MonitorUtils.getProbe(componentSpec.getReadinessCheck());
- maxContainerFailurePerComp = componentSpec.getConfiguration()
- .getPropertyInt(CONTAINER_FAILURE_THRESHOLD, 10);
- createNumCompInstances(component.getNumberOfContainers());
- }
-
- private void createNumCompInstances(long count) {
- for (int i = 0; i < count; i++) {
- createOneCompInstance();
- }
- }
-
- private void createOneCompInstance() {
- ComponentInstanceId id =
- new ComponentInstanceId(instanceIdCounter.getAndIncrement(),
- componentSpec.getName());
- ComponentInstance instance = new ComponentInstance(this, id);
- compInstances.put(id, instance);
- pendingInstances.add(instance);
- }
-
- private static class FlexComponentTransition implements
- MultipleArcTransition<Component, ComponentEvent, ComponentState> {
- // For flex up, go to FLEXING state
- // For flex down, go to STABLE state
- @Override
- public ComponentState transition(Component component,
- ComponentEvent event) {
- component.setDesiredContainers((int)event.getDesired());
- if (!component.areDependenciesReady()) {
- LOG.info("[FLEX COMPONENT {}]: Flex deferred because dependencies not"
- + " satisfied.", component.getName());
- return component.getState();
- }
- if (component.getState() == INIT) {
- // This happens on init
- LOG.info("[INIT COMPONENT " + component.getName() + "]: " + event
- .getDesired() + " instances.");
- component.requestContainers(event.getDesired());
- return FLEXING;
- }
- long before = component.getComponentSpec().getNumberOfContainers();
- long delta = event.getDesired() - before;
- component.getComponentSpec().setNumberOfContainers(event.getDesired());
- if (delta > 0) {
- // Scale up
- LOG.info("[FLEX UP COMPONENT " + component.getName() + "]: scaling up from "
- + before + " to " + event.getDesired());
- component.requestContainers(delta);
- component.createNumCompInstances(delta);
- return FLEXING;
- } else if (delta < 0){
- delta = 0 - delta;
- // scale down
- LOG.info("[FLEX DOWN COMPONENT " + component.getName()
- + "]: scaling down from " + before + " to " + event.getDesired());
- List<ComponentInstance> list =
- new ArrayList<>(component.compInstances.values());
-
- // sort in Most recent -> oldest order, destroy most recent ones.
- Collections.sort(list, Collections.reverseOrder());
- for (int i = 0; i < delta; i++) {
- ComponentInstance instance = list.get(i);
- // remove the instance
- component.compInstances.remove(instance.getCompInstanceId());
- component.pendingInstances.remove(instance);
- component.componentMetrics.containersFailed.incr();
- component.componentMetrics.containersRunning.decr();
- // decrement id counter
- component.instanceIdCounter.decrementAndGet();
- instance.destroy();
- }
- return STABLE;
- } else {
- LOG.info("[FLEX COMPONENT " + component.getName() + "]: already has " +
- event.getDesired() + " instances, ignoring");
- return STABLE;
- }
- }
- }
-
- private static class ContainerAllocatedTransition extends BaseTransition {
- @Override
- public void transition(Component component, ComponentEvent event) {
- component.assignContainerToCompInstance(event.getContainer());
- }
- }
-
- private static class ContainerStartedTransition implements
- MultipleArcTransition<Component,ComponentEvent,ComponentState> {
-
- @Override public ComponentState transition(Component component,
- ComponentEvent event) {
- component.compInstanceDispatcher.getEventHandler().handle(
- new ComponentInstanceEvent(event.getInstance().getContainerId(),
- STARTED));
- component.incRunningContainers();
- return checkIfStable(component);
- }
- }
-
- private static ComponentState checkIfStable(Component component) {
- // if desired == running
- if (component.componentMetrics.containersRunning.value() == component
- .getComponentSpec().getNumberOfContainers()) {
- return STABLE;
- } else {
- return FLEXING;
- }
- }
-
- private static class ContainerCompletedTransition extends BaseTransition {
- @Override
- public void transition(Component component, ComponentEvent event) {
- component.updateMetrics(event.getStatus());
-
- // add back to pending list
- component.pendingInstances.add(event.getInstance());
- LOG.info(
- "[COMPONENT {}]: {} completed, num pending comp instances increased to {}.",
- component.getName(), event.getStatus().getContainerId(),
- component.pendingInstances.size());
- component.compInstanceDispatcher.getEventHandler().handle(
- new ComponentInstanceEvent(event.getStatus().getContainerId(),
- STOP).setStatus(event.getStatus()));
- }
- }
-
- public ServiceMetrics getCompMetrics () {
- return componentMetrics;
- }
-
- private void assignContainerToCompInstance(Container container) {
- if (pendingInstances.size() == 0) {
- LOG.info(
- "[COMPONENT {}]: No pending component instance left, release surplus container {}",
- getName(), container.getId());
- scheduler.getAmRMClient().releaseAssignedContainer(container.getId());
- componentMetrics.surplusContainers.incr();
- scheduler.getServiceMetrics().surplusContainers.incr();
- return;
- }
- ComponentInstance instance = pendingInstances.remove(0);
- LOG.info(
- "[COMPONENT {}]: {} allocated, num pending component instances reduced to {}",
- getName(), container.getId(), pendingInstances.size());
- instance.setContainer(container);
- scheduler.addLiveCompInstance(container.getId(), instance);
- LOG.info(
- "[COMPONENT {}]: Assigned {} to component instance {} and launch on host {} ",
- getName(), container.getId(), instance.getCompInstanceName(),
- container.getNodeId());
- scheduler.getContainerLaunchService()
- .launchCompInstance(scheduler.getApp(), instance, container);
- }
-
- @SuppressWarnings({ "unchecked" })
- public void requestContainers(long count) {
- Resource resource = Resource
- .newInstance(componentSpec.getResource().getMemoryMB(),
- componentSpec.getResource().getCpus());
-
- for (int i = 0; i < count; i++) {
- //TODO Once YARN-5468 is done, use that for anti-affinity
- ContainerRequest request =
- ContainerRequest.newBuilder().capability(resource).priority(priority)
- .allocationRequestId(allocateId).relaxLocality(true).build();
- amrmClient.addContainerRequest(request);
- }
- }
-
- private void setDesiredContainers(int n) {
- int delta = n - scheduler.getServiceMetrics().containersDesired.value();
- if (delta > 0) {
- scheduler.getServiceMetrics().containersDesired.incr(delta);
- } else {
- scheduler.getServiceMetrics().containersDesired.decr(delta);
- }
- componentMetrics.containersDesired.set(n);
- }
-
-
-
- private void updateMetrics(ContainerStatus status) {
- switch (status.getExitStatus()) {
- case SUCCESS:
- componentMetrics.containersSucceeded.incr();
- scheduler.getServiceMetrics().containersSucceeded.incr();
- return;
- case PREEMPTED:
- componentMetrics.containersPreempted.incr();
- scheduler.getServiceMetrics().containersPreempted.incr();
- break;
- case DISKS_FAILED:
- componentMetrics.containersDiskFailure.incr();
- scheduler.getServiceMetrics().containersDiskFailure.incr();
- break;
- default:
- break;
- }
-
- // containersFailed include preempted, disks_failed etc.
- componentMetrics.containersFailed.incr();
- scheduler.getServiceMetrics().containersFailed.incr();
-
- // dec running container
- decRunningContainers();
-
- if (Apps.shouldCountTowardsNodeBlacklisting(status.getExitStatus())) {
- String host = scheduler.getLiveInstances().get(status.getContainerId())
- .getNodeId().getHost();
- failureTracker.incNodeFailure(host);
- currentContainerFailure++ ;
- }
- }
-
- public boolean areDependenciesReady() {
- List<String> dependencies = componentSpec.getDependencies();
- if (SliderUtils.isEmpty(dependencies)) {
- return true;
- }
- for (String dependency : dependencies) {
- Component dependentComponent =
- scheduler.getAllComponents().get(dependency);
- if (dependentComponent == null) {
- LOG.error("Couldn't find dependency {} for {} (should never happen)",
- dependency, getName());
- continue;
- }
- if (dependentComponent.getNumReadyInstances() < dependentComponent
- .getNumDesiredInstances()) {
- LOG.info("[COMPONENT {}]: Dependency {} not satisfied, only {} of {}"
- + " instances are ready.", getName(), dependency,
- dependentComponent.getNumReadyInstances(),
- dependentComponent.getNumDesiredInstances());
- return false;
- }
- }
- return true;
- }
-
- private void incRunningContainers() {
- componentMetrics.containersRunning.incr();
- scheduler.getServiceMetrics().containersRunning.incr();
- }
-
- public void incContainersReady() {
- componentMetrics.containersReady.incr();
- }
-
- public void decContainersReady() {
- componentMetrics.containersReady.decr();
- }
-
- private void decRunningContainers() {
- componentMetrics.containersRunning.decr();
- scheduler.getServiceMetrics().containersRunning.decr();
- }
-
- public int getNumReadyInstances() {
- return componentMetrics.containersReady.value();
- }
-
- public int getNumRunningInstances() {
- return componentMetrics.containersRunning.value();
- }
-
- public int getNumDesiredInstances() {
- return componentMetrics.containersDesired.value();
- }
-
- public Map<ComponentInstanceId, ComponentInstance> getAllComponentInstances() {
- return compInstances;
- }
-
- public org.apache.hadoop.yarn.service.api.records.Component getComponentSpec() {
- return this.componentSpec;
- }
-
- public void resetCompFailureCount() {
- LOG.info("[COMPONENT {}]: Reset container failure count from {} to 0.",
- getName(), currentContainerFailure);
- currentContainerFailure = 0;
- failureTracker.resetContainerFailures();
- }
-
- public Probe getProbe() {
- return probe;
- }
-
- public Priority getPriority() {
- return priority;
- }
-
- public long getAllocateId() {
- return allocateId;
- }
-
- public String getName () {
- return componentSpec.getName();
- }
-
- public ComponentState getState() {
- this.readLock.lock();
-
- try {
- return this.stateMachine.getCurrentState();
- } finally {
- this.readLock.unlock();
- }
- }
- public ServiceScheduler getScheduler() {
- return scheduler;
- }
-
- @Override
- public void handle(ComponentEvent event) {
- try {
- writeLock.lock();
- ComponentState oldState = getState();
- try {
- stateMachine.doTransition(event.getType(), event);
- } catch (InvalidStateTransitionException e) {
- LOG.error(MessageFormat.format("[COMPONENT {0}]: Invalid event {1} at {2}",
- componentSpec.getName(), event.getType(), oldState), e);
- }
- if (oldState != getState()) {
- LOG.info("[COMPONENT {}] Transitioned from {} to {} on {} event.",
- componentSpec.getName(), oldState, getState(), event.getType());
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- private static class BaseTransition implements
- SingleArcTransition<Component, ComponentEvent> {
-
- @Override public void transition(Component component,
- ComponentEvent event) {
- }
- }
-
- public ServiceContext getContext() {
- return context;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
deleted file mode 100644
index ed892dd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.component;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-
-public class ComponentEvent extends AbstractEvent<ComponentEventType> {
- private long desired;
- private final String name;
- private final ComponentEventType type;
- private Container container;
- private ComponentInstance instance;
- private ContainerStatus status;
-
- public ComponentEvent(String name, ComponentEventType type) {
- super(type);
- this.name = name;
- this.type = type;
- }
-
- public String getName() {
- return name;
- }
-
- public ComponentEventType getType() {
- return type;
- }
-
- public long getDesired() {
- return desired;
- }
-
- public ComponentEvent setDesired(long desired) {
- this.desired = desired;
- return this;
- }
-
- public Container getContainer() {
- return container;
- }
-
- public ComponentEvent setContainer(Container container) {
- this.container = container;
- return this;
- }
-
- public ComponentInstance getInstance() {
- return instance;
- }
-
- public ComponentEvent setInstance(ComponentInstance instance) {
- this.instance = instance;
- return this;
- }
-
- public ContainerStatus getStatus() {
- return status;
- }
-
- public ComponentEvent setStatus(ContainerStatus status) {
- this.status = status;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
deleted file mode 100644
index 6729699..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.component;
-
-public enum ComponentEventType {
- FLEX,
- CONTAINER_ALLOCATED,
- CONTAINER_STARTED,
- CONTAINER_COMPLETED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
deleted file mode 100644
index a5f9ff4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.component;
-
-public enum ComponentState {
- INIT,
- FLEXING,
- STABLE
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
deleted file mode 100644
index cbbb206..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-public interface RestApiConstants {
-
- // Rest endpoints
- String CONTEXT_ROOT = "/services/v1";
- String VERSION = "/version";
- String APP_ROOT_PATH = "/applications";
- String APP_PATH = "/applications/{app_name}";
- String COMPONENT_PATH = "/applications/{app_name}/components/{component_name}";
-
- // Query param
- String APP_NAME = "app_name";
- String COMPONENT_NAME = "component_name";
-
- String DEFAULT_COMPONENT_NAME = "default";
-
- String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
- String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
- Long DEFAULT_UNLIMITED_LIFETIME = -1l;
-
- Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
- Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
- Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
- Integer ERROR_CODE_APP_NAME_INVALID = 404004;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
deleted file mode 100644
index bdef600..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
-
-public interface SliderExitCodes extends LauncherExitCodes {
-
- /**
- * starting point for exit codes; not an exception itself
- */
- int _EXIT_CODE_BASE = 64;
-
- /**
- * service entered the failed state: {@value}
- */
- int EXIT_YARN_SERVICE_FAILED = 65;
-
- /**
- * service was killed: {@value}
- */
- int EXIT_YARN_SERVICE_KILLED = 66;
-
- /**
- * timeout on monitoring client: {@value}
- */
- int EXIT_TIMED_OUT = 67;
-
- /**
- * service finished with an error: {@value}
- */
- int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
-
- /**
- * the application instance is unknown: {@value}
- */
- int EXIT_UNKNOWN_INSTANCE = 69;
-
- /**
- * the application instance is in the wrong state for that operation: {@value}
- */
- int EXIT_BAD_STATE = 70;
-
- /**
- * A spawned master process failed
- */
- int EXIT_PROCESS_FAILED = 71;
-
- /**
- * The instance failed -too many containers were
- * failing or some other threshold was reached
- */
- int EXIT_DEPLOYMENT_FAILED = 72;
-
- /**
- * The application is live -and the requested operation
- * does not work if the cluster is running
- */
- int EXIT_APPLICATION_IN_USE = 73;
-
- /**
- * There already is an application instance of that name
- * when an attempt is made to create a new instance
- */
- int EXIT_INSTANCE_EXISTS = 75;
-
- /**
- * Exit code when the configurations in valid/incomplete: {@value}
- */
- int EXIT_BAD_CONFIGURATION = 77;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
deleted file mode 100644
index 33fc671..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-
-public class YarnServiceConf {
-
- // Retry settings for the ServiceClient to talk to Service AppMaster
- public static final String CLIENT_AM_RETRY_MAX_WAIT_MS = "yarn.service.client-am.retry.max-wait-ms";
- public static final String CLIENT_AM_RETRY_MAX_INTERVAL_MS = "yarn.service.client-am.retry-interval-ms";
-
- // Retry settings for container failures
- public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
- public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval";
-
- public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
- public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
- public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
-
- public static final String YARN_QUEUE = "yarn.service.queue";
-
- /**
- * The yarn service base path:
- * Defaults to HomeDir/.yarn/
- */
- public static final String YARN_SERVICE_BASE_PATH = "yarn.service.base.path";
-
- //TODO rename
- /** Declare that a keytab must be provided */
- public static final String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
- public static final String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
- public static final String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
- public static final String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
-
- /**
- * maximum number of failed containers (in a single component)
- * before the app exits
- */
- public static final String CONTAINER_FAILURE_THRESHOLD =
- "yarn.service.container-failure-per-component.threshold";
- /**
- * Maximum number of container failures on a node before the node is blacklisted
- */
- public static final String NODE_BLACKLIST_THRESHOLD =
- "yarn.service.node-blacklist.threshold";
-
- /**
- * The failure count for CONTAINER_FAILURE_THRESHOLD and NODE_BLACKLIST_THRESHOLD
- * gets reset periodically, the unit is seconds.
- */
- public static final String CONTAINER_FAILURE_WINDOW =
- "yarn.service.failure-count-reset.window";
-
- /**
- * interval between readiness checks.
- */
- public static final String READINESS_CHECK_INTERVAL = "yarn.service.readiness-check-interval.seconds";
- public static final int DEFAULT_READINESS_CHECK_INTERVAL = 30; // seconds
-
- /**
- * Get long value for the property. First get from the userConf, if not
- * present, get from systemConf.
- *
- * @param name name of the property
- * @param defaultValue default value of the property, if it is not defined in
- * userConf and systemConf.
- * @param userConf Configuration provided by client in the JSON definition
- * @param systemConf The YarnConfiguration in the system.
- * @return long value for the property
- */
- public static long getLong(String name, long defaultValue,
- Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
- return userConf.getPropertyLong(name, systemConf.getLong(name, defaultValue));
- }
-
- public static int getInt(String name, int defaultValue,
- Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
- return userConf.getPropertyInt(name, systemConf.getInt(name, defaultValue));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
deleted file mode 100644
index cbcba82..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-public interface YarnServiceConstants {
-
- /**
- * The path under which cluster and temp data are stored
- */
- String SERVICE_BASE_DIRECTORY = ".yarn";
-
- /**
- * The paths under which Service AM dependency libraries are stored
- */
- String DEPENDENCY_LOCALIZED_DIR_LINK = "service_dep";
- String DEPENDENCY_DIR = "/yarn-services/%s/";
- String DEPENDENCY_TAR_GZ_FILE_NAME = "service-dep";
- String DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
- String DEPENDENCY_DIR_PERMISSIONS = "755";
-
- /**
- * Application type for YARN service
- */
- String APP_TYPE = "yarn-service";
-
- String KEYTAB_DIR = "keytabs";
- String RESOURCE_DIR = "resources";
-
-
- String SERVICES_DIRECTORY = "services";
-
- /**
- * JVM property to define the service lib directory;
- * this is set by the yarn.sh script
- */
- String PROPERTY_LIB_DIR = "service.libdir";
-
- /**
- * name of generated dir for this conf
- */
- String SUBMITTED_CONF_DIR = "conf";
-
- /**
- * Service AM log4j file name
- */
- String YARN_SERVICE_LOG4J_FILENAME = "yarnservice-log4j.properties";
-
- /**
- * Log4j sysprop to name the resource
- */
- String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
-
- /**
- * sysprop for Service AM log4j directory
- */
- String SYSPROP_LOG_DIR = "LOG_DIR";
-
- String TMP_DIR_PREFIX = "tmp";
-
-
- String SERVICE_CORE_JAR = "yarn-service-core.jar";
-
- String STDOUT_AM = "serviceam-out.txt";
- String STDERR_AM = "serviceam-err.txt";
-
- String HADOOP_USER_NAME = "HADOOP_USER_NAME";
-
- String APP_CONF_DIR = "conf";
-
- String APP_LIB_DIR = "lib";
-
- String OUT_FILE = "stdout.txt";
- String ERR_FILE = "stderr.txt";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
deleted file mode 100644
index e4eae20..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
-import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
-
-/**
- * Launcher of applications: base class
- */
-public class AbstractLauncher {
- private static final Logger log =
- LoggerFactory.getLogger(AbstractLauncher.class);
- public static final String CLASSPATH = "CLASSPATH";
- /**
- * Filesystem to use for the launch
- */
- protected final CoreFileSystem coreFileSystem;
- /**
- * Env vars; set up at final launch stage
- */
- protected final Map<String, String> envVars = new HashMap<>();
- protected final ContainerLaunchContext containerLaunchContext =
- Records.newRecord(ContainerLaunchContext.class);
- protected final List<String> commands = new ArrayList<>(20);
- protected final Map<String, LocalResource> localResources = new HashMap<>();
- protected final Map<String, String> mountPaths = new HashMap<>();
- private final Map<String, ByteBuffer> serviceData = new HashMap<>();
- // security
- protected final Credentials credentials;
- protected boolean yarnDockerMode = false;
- protected String dockerImage;
- protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
- protected String dockerHostname;
- protected String runPrivilegedContainer;
-
-
- /**
- * Create instance.
- * @param coreFileSystem filesystem
- * @param credentials initial set of credentials -null is permitted
- */
- public AbstractLauncher(
- CoreFileSystem coreFileSystem,
- Credentials credentials) {
- this.coreFileSystem = coreFileSystem;
- this.credentials = credentials != null ? credentials: new Credentials();
- }
-
- public void setYarnDockerMode(boolean yarnDockerMode){
- this.yarnDockerMode = yarnDockerMode;
- }
-
- /**
- * Get the env vars to work on
- * @return env vars
- */
- public Map<String, String> getEnv() {
- return envVars;
- }
-
- /**
- * Get the launch commands.
- * @return the live list of commands
- */
- public List<String> getCommands() {
- return commands;
- }
-
- public void addLocalResource(String subPath, LocalResource resource) {
- localResources.put(subPath, resource);
- }
-
- public void addLocalResource(String subPath, LocalResource resource, String mountPath) {
- localResources.put(subPath, resource);
- mountPaths.put(subPath, mountPath);
- }
-
- /**
- * Accessor to the credentials
- * @return the credentials associated with this launcher
- */
- public Credentials getCredentials() {
- return credentials;
- }
-
-
- public void addCommand(String cmd) {
- commands.add(cmd);
- }
-
- /**
- * Complete the launch context (copy in env vars, etc).
- * @return the container to launch
- */
- public ContainerLaunchContext completeContainerLaunch() throws IOException {
-
- String cmdStr = SliderUtils.join(commands, " ", false);
- log.debug("Completed setting up container command {}", cmdStr);
- containerLaunchContext.setCommands(commands);
-
- //env variables
- if (log.isDebugEnabled()) {
- log.debug("Environment variables");
- for (Map.Entry<String, String> envPair : envVars.entrySet()) {
- log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
- }
- }
- containerLaunchContext.setEnvironment(envVars);
-
- //service data
- if (log.isDebugEnabled()) {
- log.debug("Service Data size");
- for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) {
- log.debug("\"{}\"=> {} bytes of data", entry.getKey(),
- entry.getValue().array().length);
- }
- }
- containerLaunchContext.setServiceData(serviceData);
-
- // resources
- dumpLocalResources();
- containerLaunchContext.setLocalResources(localResources);
-
- //tokens
- log.debug("{} tokens", credentials.numberOfTokens());
- containerLaunchContext.setTokens(CredentialUtils.marshallCredentials(
- credentials));
-
- if(yarnDockerMode){
- Map<String, String> env = containerLaunchContext.getEnvironment();
- env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", dockerNetwork);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME",
- dockerHostname);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
- StringBuilder sb = new StringBuilder();
- for (Entry<String,String> mount : mountPaths.entrySet()) {
- if (sb.length() > 0) {
- sb.append(",");
- }
- sb.append(mount.getKey());
- sb.append(":");
- sb.append(mount.getValue());
- }
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString());
- log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString());
- }
-
- return containerLaunchContext;
- }
-
- public void setRetryContext(int maxRetries, int retryInterval) {
- ContainerRetryContext retryContext = ContainerRetryContext
- .newInstance(ContainerRetryPolicy.RETRY_ON_ALL_ERRORS, null, maxRetries,
- retryInterval);
- containerLaunchContext.setContainerRetryContext(retryContext);
- }
-
- /**
- * Dump local resources at debug level
- */
- private void dumpLocalResources() {
- if (log.isDebugEnabled()) {
- log.debug("{} resources: ", localResources.size());
- for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
-
- String key = entry.getKey();
- LocalResource val = entry.getValue();
- log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
- }
- }
- }
-
- /**
- * This is critical for an insecure cluster -it passes
- * down the username to YARN, and so gives the code running
- * in containers the rights it needs to work with
- * data.
- * @throws IOException problems working with current user
- */
- protected void propagateUsernameInInsecureCluster() throws IOException {
- //insecure cluster: propagate user name via env variable
- String userName = UserGroupInformation.getCurrentUser().getUserName();
- envVars.put(YarnServiceConstants.HADOOP_USER_NAME, userName);
- }
-
- /**
- * Utility method to set up the classpath
- * @param classpath classpath to use
- */
- public void setClasspath(ClasspathConstructor classpath) {
- setEnv(CLASSPATH, classpath.buildClasspath());
- }
-
- /**
- * Set an environment variable in the launch context
- * @param var variable name
- * @param value value (must be non null)
- */
- public void setEnv(String var, String value) {
- Preconditions.checkArgument(var != null, "null variable name");
- Preconditions.checkArgument(value != null, "null value");
- envVars.put(var, value);
- }
-
-
- public void putEnv(Map<String, String> map) {
- envVars.putAll(map);
- }
-
-
- public void setDockerImage(String dockerImage) {
- this.dockerImage = dockerImage;
- }
-
- public void setDockerNetwork(String dockerNetwork) {
- this.dockerNetwork = dockerNetwork;
- }
-
- public void setDockerHostname(String dockerHostname) {
- this.dockerHostname = dockerHostname;
- }
-
- public void setRunPrivilegedContainer(boolean runPrivilegedContainer) {
- if (runPrivilegedContainer) {
- this.runPrivilegedContainer = Boolean.toString(true);
- } else {
- this.runPrivilegedContainer = Boolean.toString(false);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
deleted file mode 100644
index 22b3877..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * build a classpath -allows for entries to be injected in front of
- * YARN classpath as well as behind, adds appropriate separators,
- * extraction of local classpath, etc.
- */
-public class ClasspathConstructor {
-
- public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
- private final List<String> pathElements = new ArrayList<>();
-
- public ClasspathConstructor() {
- }
-
-
- /**
- * Get the list of JARs from the YARN settings
- * @param config configuration
- */
- public List<String> yarnApplicationClasspath(Configuration config) {
- String[] cp = config.getTrimmedStrings(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH);
- return cp != null ? Arrays.asList(cp) : new ArrayList<String>(0);
-
- }
-
-
- @Override
- public String toString() {
- return buildClasspath();
- }
-
- public String buildClasspath() {
- return SliderUtils.join(pathElements,
- CLASS_PATH_SEPARATOR,
- false);
- }
-
- /**
- * Get a copy of the path list
- * @return the JARs
- */
- public List<String> getPathElements() {
- return Collections.unmodifiableList(pathElements);
- }
-
- /**
- * Append an entry
- * @param path path
- */
- public void append(String path) {
- pathElements.add(path);
- }
-
- /**
- * Insert a path at the front of the list. This places it ahead of
- * the standard YARN artifacts
- * @param path path to the JAR. Absolute or relative -on the target
- * system
- */
- public void insert(String path) {
- pathElements.add(0, path);
- }
-
- public void appendAll(Collection<String> paths) {
- pathElements.addAll(paths);
- }
-
- public void insertAll(Collection<String> paths) {
- pathElements.addAll(0, paths);
- }
-
-
- public void addLibDir(String pathToLibDir) {
- append(buildLibDir(pathToLibDir));
- }
-
- public void insertLibDir(String pathToLibDir) {
- insert(buildLibDir(pathToLibDir));
- }
-
- public void addClassDirectory(String pathToDir) {
- append(appendDirectoryTerminator(pathToDir));
- }
-
- public void insertClassDirectory(String pathToDir) {
- insert(buildLibDir(appendDirectoryTerminator(pathToDir)));
- }
-
-
- public void addRemoteClasspathEnvVar() {
- append(ApplicationConstants.Environment.CLASSPATH.$$());
- }
-
-
- public void insertRemoteClasspathEnvVar() {
- append(ApplicationConstants.Environment.CLASSPATH.$$());
- }
-
-
- /**
- * Build a lib dir path
- * @param pathToLibDir path to the directory; may or may not end with a
- * trailing space
- * @return a path to a lib dir that is compatible with the java classpath
- */
- public String buildLibDir(String pathToLibDir) {
- String dir = appendDirectoryTerminator(pathToLibDir);
- dir += "*";
- return dir;
- }
-
- private String appendDirectoryTerminator(String pathToLibDir) {
- String dir = pathToLibDir.trim();
- if (!dir.endsWith("/")) {
- dir += "/";
- }
- return dir;
- }
-
- /**
- * Split a classpath. This uses the local path separator so MUST NOT
- * be used to work with remote classpaths
- * @param localpath local path
- * @return a splite
- */
- public Collection<String> splitClasspath(String localpath) {
- String separator = System.getProperty("path.separator");
- return StringUtils.getStringCollection(localpath, separator);
- }
-
- /**
- * Get the local JVM classpath split up
- * @return the list of entries on the JVM classpath env var
- */
- public Collection<String> localJVMClasspath() {
- return splitClasspath(System.getProperty("java.class.path"));
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
new file mode 100644
index 0000000..b742553
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test class for ServiceTimelinePublisher.
+ */
+public class TestServiceTimelinePublisher {
+ private TimelineV2Client timelineClient;
+ private Configuration config;
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+ private static String SERVICE_NAME = "HBASE";
+ private static String SERVICEID = "application_1490093646524_0005";
+ private static String ARTIFACTID = "ARTIFACTID";
+ private static String COMPONENT_NAME = "DEFAULT";
+ private static String CONTAINER_ID =
+ "container_e02_1490093646524_0005_01_000001";
+ private static String CONTAINER_IP =
+ "localhost";
+ private static String CONTAINER_HOSTNAME =
+ "cnl124-localhost.site";
+ private static String CONTAINER_BAREHOST =
+ "localhost.com";
+
+ @Before
+ public void setUp() throws Exception {
+ config = new Configuration();
+ config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+ timelineClient =
+ new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
+ serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+ timelineClient.init(config);
+ serviceTimelinePublisher.init(config);
+ timelineClient.start();
+ serviceTimelinePublisher.start();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (serviceTimelinePublisher != null) {
+ serviceTimelinePublisher.stop();
+ }
+ if (timelineClient != null) {
+ timelineClient.stop();
+ }
+ }
+
+ @Test
+ public void testServiceAttemptEntity() {
+ Service service = createMockApplication();
+ serviceTimelinePublisher
+ .serviceAttemptRegistered(service, new YarnConfiguration());
+
+ Collection<TimelineEntity> lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ // 2 entities because during registration component also registered.
+ assertEquals(2, lastPublishedEntities.size());
+ for (TimelineEntity timelineEntity : lastPublishedEntities) {
+ if (timelineEntity.getType() == ServiceTimelineEntityType.COMPONENT
+ .toString()) {
+ verifyComponentTimelineEntity(timelineEntity);
+ } else {
+ verifyServiceAttemptTimelineEntity(timelineEntity, null, true);
+ }
+ }
+
+ ServiceContext context = new ServiceContext();
+ context.attemptId = ApplicationAttemptId
+ .newInstance(ApplicationId.fromString(service.getId()), 1);
+ String exitDiags = "service killed";
+ serviceTimelinePublisher.serviceAttemptUnregistered(context, exitDiags);
+ lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ for (TimelineEntity timelineEntity : lastPublishedEntities) {
+ if (timelineEntity.getType() == ServiceTimelineEntityType.SERVICE_ATTEMPT
+ .toString()) {
+ verifyServiceAttemptTimelineEntity(timelineEntity, exitDiags,
+ false);
+ }
+ }
+ }
+
+ @Test
+ public void testComponentInstanceEntity() {
+ Container container = new Container();
+ container.id(CONTAINER_ID).ip(CONTAINER_IP).bareHost(CONTAINER_BAREHOST)
+ .hostname(CONTAINER_HOSTNAME).state(ContainerState.RUNNING_BUT_UNREADY)
+ .launchTime(new Date());
+ ComponentInstanceId id = new ComponentInstanceId(0, COMPONENT_NAME);
+ ComponentInstance instance = mock(ComponentInstance.class);
+ when(instance.getCompName()).thenReturn(COMPONENT_NAME);
+ when(instance.getCompInstanceName()).thenReturn("comp_instance_name");
+ serviceTimelinePublisher.componentInstanceStarted(container,
+ instance);
+
+ Collection<TimelineEntity> lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ assertEquals(1, lastPublishedEntities.size());
+ TimelineEntity entity = lastPublishedEntities.iterator().next();
+
+ assertEquals(1, entity.getEvents().size());
+ assertEquals(CONTAINER_ID, entity.getId());
+ assertEquals(CONTAINER_BAREHOST,
+ entity.getInfo().get(ServiceTimelineMetricsConstants.BARE_HOST));
+ assertEquals(COMPONENT_NAME,
+ entity.getInfo().get(ServiceTimelineMetricsConstants.COMPONENT_NAME));
+ assertEquals(ContainerState.RUNNING_BUT_UNREADY.toString(),
+ entity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
+
+ // updated container state
+ container.setState(ContainerState.READY);
+ serviceTimelinePublisher.componentInstanceUpdated(container);
+ lastPublishedEntities =
+ ((DummyTimelineClient) timelineClient).getLastPublishedEntities();
+ assertEquals(1, lastPublishedEntities.size());
+ entity = lastPublishedEntities.iterator().next();
+ assertEquals(2, entity.getEvents().size());
+ assertEquals(ContainerState.READY.toString(),
+ entity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
+
+ }
+
+ private void verifyServiceAttemptTimelineEntity(TimelineEntity timelineEntity,
+ String message, boolean isRegistedEntity) {
+ assertEquals(SERVICEID, timelineEntity.getId());
+ assertEquals(SERVICE_NAME,
+ timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.NAME));
+ if (isRegistedEntity) {
+ assertEquals(ServiceState.STARTED.toString(),
+ timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE));
+ assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString(),
+ timelineEntity.getEvents().iterator().next().getId());
+ } else {
+ assertEquals("ENDED",
+ timelineEntity.getInfo().get(ServiceTimelineMetricsConstants.STATE).toString());
+ assertEquals(message, timelineEntity.getInfo()
+ .get(ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO));
+ assertEquals(2, timelineEntity.getEvents().size());
+ assertEquals(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString(),
+ timelineEntity.getEvents().iterator().next().getId());
+ }
+ }
+
+ private void verifyComponentTimelineEntity(TimelineEntity entity) {
+ Map<String, Object> info = entity.getInfo();
+ assertEquals("DEFAULT", entity.getId());
+ assertEquals(ARTIFACTID,
+ info.get(ServiceTimelineMetricsConstants.ARTIFACT_ID));
+ assertEquals("DOCKER",
+ info.get(ServiceTimelineMetricsConstants.ARTIFACT_TYPE));
+ assertEquals("medium",
+ info.get(ServiceTimelineMetricsConstants.RESOURCE_PROFILE));
+ assertEquals(1, info.get(ServiceTimelineMetricsConstants.RESOURCE_CPU));
+ assertEquals("1024",
+ info.get(ServiceTimelineMetricsConstants.RESOURCE_MEMORY));
+ assertEquals("sleep 1",
+ info.get(ServiceTimelineMetricsConstants.LAUNCH_COMMAND));
+ assertEquals("false",
+ info.get(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER));
+ assertEquals("label",
+ info.get(ServiceTimelineMetricsConstants.PLACEMENT_POLICY));
+ }
+
+ private static Service createMockApplication() {
+ Service service = mock(Service.class);
+
+ when(service.getId()).thenReturn(SERVICEID);
+ when(service.getLaunchTime()).thenReturn(new Date());
+ when(service.getState()).thenReturn(ServiceState.STARTED);
+ when(service.getName()).thenReturn(SERVICE_NAME);
+ when(service.getConfiguration()).thenReturn(
+ new org.apache.hadoop.yarn.service.api.records.Configuration());
+
+ Component component = mock(Component.class);
+ Artifact artifact = new Artifact();
+ artifact.setId(ARTIFACTID);
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory(1024 + "");
+ resource.setProfile("medium");
+ when(component.getArtifact()).thenReturn(artifact);
+ when(component.getName()).thenReturn(COMPONENT_NAME);
+ when(component.getResource()).thenReturn(resource);
+ when(component.getLaunchCommand()).thenReturn("sleep 1");
+ PlacementPolicy placementPolicy = new PlacementPolicy();
+ placementPolicy.setLabel("label");
+ when(component.getPlacementPolicy()).thenReturn(placementPolicy);
+ when(component.getConfiguration()).thenReturn(
+ new org.apache.hadoop.yarn.service.api.records.Configuration());
+ List<Component> components = new ArrayList<Component>();
+ components.add(component);
+
+ when(service.getComponents()).thenReturn(components);
+ return service;
+ }
+
+ protected static class DummyTimelineClient extends TimelineV2ClientImpl {
+ private Map<Identifier, TimelineEntity> lastPublishedEntities =
+ new HashMap<>();
+
+ public DummyTimelineClient(ApplicationId appId) {
+ super(appId);
+ }
+
+ @Override
+ public void putEntitiesAsync(TimelineEntity... entities)
+ throws IOException, YarnException {
+ putEntities(entities);
+ }
+
+ @Override
+ public void putEntities(TimelineEntity... entities)
+ throws IOException, YarnException {
+ for (TimelineEntity timelineEntity : entities) {
+ TimelineEntity entity =
+ lastPublishedEntities.get(timelineEntity.getIdentifier());
+ if (entity == null) {
+ lastPublishedEntities.put(timelineEntity.getIdentifier(),
+ timelineEntity);
+ } else {
+ entity.addMetrics(timelineEntity.getMetrics());
+ entity.addEvents(timelineEntity.getEvents());
+ entity.addInfo(timelineEntity.getInfo());
+ entity.addConfigs(timelineEntity.getConfigs());
+ entity.addRelatesToEntities(timelineEntity.getRelatesToEntities());
+ entity
+ .addIsRelatedToEntities(timelineEntity.getIsRelatedToEntities());
+ }
+ }
+ }
+
+ public Collection<TimelineEntity> getLastPublishedEntities() {
+ return lastPublishedEntities.values();
+ }
+
+ public void reset() {
+ lastPublishedEntities = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/example-app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/example-app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/example-app.json
new file mode 100644
index 0000000..5dfbd64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/example-app.json
@@ -0,0 +1,15 @@
+{
+ "name": "example-app",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
new file mode 100644
index 0000000..d7e2fd0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
@@ -0,0 +1,72 @@
+{
+ "name": "app-1",
+ "lifetime": "3600",
+ "launch_command": "sleep 3600",
+ "configuration": {
+ "properties": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "files": [
+ {
+ "type": "PROPERTIES",
+ "dest_file": "file1",
+ "props": {
+ "k1": "v1",
+ "k2": "v2"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "file2",
+ "props": {
+ "k3": "v3"
+ }
+ }
+ ]
+ },
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ },
+ "number_of_containers": 2,
+ "components": [
+ {
+ "name": "simple",
+ "configuration": {
+ "files": [
+ {
+ "type": "PROPERTIES",
+ "dest_file": "file1",
+ "props": {
+ "k1": "overridden"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "master",
+ "configuration": {
+ "properties": {
+ "name": "m",
+ "g1": "overridden"
+ }
+ }
+ },
+ {
+ "name": "worker",
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "properties": {
+ "name": "worker",
+ "g1": "overridden-by-worker",
+ "timeout": "1000"
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
new file mode 100644
index 0000000..12b51e4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
@@ -0,0 +1,48 @@
+{
+ "name": "service-1",
+ "id" : "application_1503358878042_0011",
+ "lifetime": "3600",
+ "launch_command": "sleep 3600",
+ "configuration": {
+ "properties": {
+ "g1": "a",
+ "g2": "b",
+ "yarn.service.failure-count-reset.window": "60"
+ }
+ },
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ },
+ "number_of_containers": 2,
+ "components": [
+ {
+ "name": "simple"
+ },
+ {
+ "name": "master",
+ "number_of_containers": 1,
+ "configuration": {
+ "properties": {
+ "g1": "overridden",
+ "g3": "will-be-overridden",
+ "jvm.heapsize": "512M"
+ }
+ }
+ },
+ {
+ "name": "worker",
+ "number_of_containers": 5,
+ "resource": {
+ "cpus": 1,
+ "memory": "1024"
+ },
+ "configuration": {
+ "properties": {
+ "g1": "overridden-by-worker",
+ "jvm.heapsize": "512M"
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
new file mode 100644
index 0000000..73d4e7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/default.json
@@ -0,0 +1,16 @@
+{
+ "name": "default-app-1",
+ "lifetime": "3600",
+ "components" :
+ [
+ {
+ "name": "sleep",
+ "number_of_containers": 1,
+ "launch_command": "sleep 3600",
+ "resource": {
+ "cpus": 2,
+ "memory": "256"
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
new file mode 100644
index 0000000..0857f62
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external0.json
@@ -0,0 +1,8 @@
+{
+ "name": "external-0",
+ "lifetime": "3600",
+ "artifact": {
+ "type": "SERVICE",
+ "id": "app-1"
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
new file mode 100644
index 0000000..4afdb8b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external1.json
@@ -0,0 +1,30 @@
+{
+ "name": "external-1",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "simple",
+ "artifact": {
+ "type": "SERVICE",
+ "id": "app-1"
+ }
+ },
+ {
+ "name": "master",
+ "configuration": {
+ "properties": {
+ "g3": "is-overridden"
+ }
+ }
+ },
+ {
+ "name": "other",
+ "launch_command": "sleep 3600",
+ "number_of_containers": 2,
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
new file mode 100644
index 0000000..0df8e0a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external2.json
@@ -0,0 +1,22 @@
+{
+ "name": "external-2",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "ext",
+ "artifact": {
+ "type": "SERVICE",
+ "id": "external-1"
+ }
+ },
+ {
+ "name": "another",
+ "launch_command": "sleep 3600",
+ "number_of_containers": 1,
+ "resource": {
+ "cpus": 1,
+ "memory": "512"
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml
new file mode 100644
index 0000000..266caa9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+ <!-- Dummy (invalid) config file to be overwriten by TestYarnNativeServices with MiniCluster configuration. -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
new file mode 100644
index 0000000..1233804
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-yarn-applications</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <version>3.0.0-beta1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hadoop-yarn-services</artifactId>
+ <name>Apache Hadoop YARN Services</name>
+ <packaging>pom</packaging>
+
+ <properties>
+ <hadoop.common.build.dir>${basedir}/../../../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
+ </properties>
+
+ <!-- Do not add dependencies here, add them to the POM of the leaf module -->
+
+ <modules>
+ <module>hadoop-yarn-services-core</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
deleted file mode 100644
index 2814cca..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<FindBugsFilter>
- <Match>
- <Package name="org.apache.hadoop.yarn.proto" />
- </Match>
- <Match>
- <class name="org.apache.hadoop.yarn.service.utils.ServiceApiUtil" />
- <Bug pattern="MS_CANNOT_BE_FINAL" />
- </Match>
- <Match>
- <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
- <Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
- </Match>
- <Match>
- <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
- <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
- </Match>
- <Match>
- <Package name="org.apache.hadoop.yarn.service.client.params"/>
- <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
- </Match>
- <Match>
- <Package name="org.apache.hadoop.yarn.service.client.params"/>
- <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
- </Match>
- <Match>
- <Class name="org.apache.hadoop.yarn.service.client.ServiceClient"/>
- <Field name="registryClient" />
- <Bug pattern="IS2_INCONSISTENT_SYNC"/>
- </Match>
-
-</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
deleted file mode 100644
index 58c8e27..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
+++ /dev/null
@@ -1,62 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is the log4j configuration for Slider Application Master
-
-# Log rotation based on size (256MB) with a max of 20 backup files
-log4j.rootLogger=INFO, amlog
-log4j.threshhold=ALL
-log4j.appender.amlog=org.apache.log4j.RollingFileAppender
-log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
-log4j.appender.amlog.File=${LOG_DIR}/serviceam.log
-log4j.appender.amlog.MaxFileSize=256MB
-log4j.appender.amlog.MaxBackupIndex=20
-
-# log layout skips stack-trace creation operations by avoiding line numbers and method
-log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
-
-# debug edition is much more expensive
-#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# configure stderr
-# set the conversion pattern of stderr
-# Print the date in ISO 8601 format
-log4j.appender.stderr=org.apache.log4j.ConsoleAppender
-log4j.appender.stderr.Target=System.err
-log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
-
-log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
-log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
-log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
-
-# for debugging yarn-service framework
-#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
-
-# uncomment for YARN operations
-#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
-
-# uncomment this to debug security problems
-#log4j.logger.org.apache.hadoop.security=DEBUG
-
-#crank back on some noise
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-log4j.logger.org.apache.hadoop.hdfs=WARN
-
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.org.apache.curator.framework.state=ERROR
-log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
deleted file mode 100644
index c8de037..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ /dev/null
@@ -1,409 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider</artifactId>
- <version>3.0.0-beta1-SNAPSHOT</version>
- </parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider-core</artifactId>
- <packaging>jar</packaging>
- <name>Apache Hadoop YARN Slider Core</name>
-
- <properties>
- <!-- Needed for generating FindBugs warnings using parent pom -->
- <yarn.basedir>${project.parent.basedir}</yarn.basedir>
- </properties>
-
- <build>
- <!-- resources are filtered for dynamic updates. This gets build info in-->
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- <filtering>true</filtering>
- </resource>
- </resources>
-
- <plugins>
- <plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
- <executions>
- <execution>
- <id>compile-protoc</id>
- <goals>
- <goal>protoc</goal>
- </goals>
- <configuration>
- <protocVersion>${protobuf.version}</protocVersion>
- <protocCommand>${protoc.path}</protocCommand>
- <imports>
- <param>${basedir}/src/main/proto</param>
- </imports>
- <source>
- <directory>${basedir}/src/main/proto</directory>
- <includes>
- <include>ClientAMProtocol.proto</include>
- </includes>
- </source>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <environmentVariables>
- <JAVA_HOME>${java.home}</JAVA_HOME>
- </environmentVariables>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- </excludes>
- </configuration>
- </plugin>
-
- </plugins>
- </build>
- <dependencies>
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- <version>1.30</version>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-jaxrs</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-xc</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-registry</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-digester</groupId>
- <artifactId>commons-digester</artifactId>
- <version>1.8</version>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-servlets</artifactId>
- <version>3.0.1</version>
- </dependency>
-
- <!-- ======================================================== -->
- <!-- service registry -->
- <!-- ======================================================== -->
-
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
-
- <!-- ======================================================== -->
- <!-- Jersey and webapp support -->
- <!-- ======================================================== -->
-
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>javax.xml.bind</groupId>
- <artifactId>jaxb-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>jersey-guice</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <version>3.1</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.objenesis</groupId>
- <artifactId>objenesis</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-easymock</artifactId>
- <version>1.6.5</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>1.6.5</version>
- <exclusions>
- <exclusion>
- <groupId>org.javassist</groupId>
- <artifactId>javassist</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.objenesis</groupId>
- <artifactId>objenesis</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- <version>1.16</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>io.swagger</groupId>
- <artifactId>swagger-annotations</artifactId>
- <version>1.5.4</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
-
- <profiles>
- <profile>
- <id>dist</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-assemblies</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>dist</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <appendAssemblyId>false</appendAssemblyId>
- <attach>false</attach>
- <finalName>${project.artifactId}-${project.version}</finalName>
- <descriptorRefs>
- <descriptorRef>hadoop-yarn-services-dist</descriptorRef>
- </descriptorRefs>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- </profiles>
-
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml
deleted file mode 100644
index 23383c8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml
+++ /dev/null
@@ -1,47 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<assembly
- xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>distribution</id>
- <formats>
- <format>zip</format>
- </formats>
- <includeBaseDirectory>true</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${project.build.directory}</directory>
- <outputDirectory>/</outputDirectory>
- <includes>
- <include>*.jar</include>
- </includes>
- </fileSet>
- </fileSets>
- <dependencySets>
- <dependencySet>
- <scope>runtime</scope>
- <outputDirectory>/lib</outputDirectory>
- <!-- dont copy JAR into /lib-->
- <useProjectArtifact>false</useProjectArtifact>
- <!-- <includeBaseDirectory>false</includeBaseDirectory>-->
- <unpack>false</unpack>
- </dependencySet>
- </dependencySets>
-</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
deleted file mode 100644
index 516d23d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-
-import java.io.IOException;
-
-public interface ClientAMProtocol {
- FlexComponentsResponseProto flexComponents(FlexComponentsRequestProto request)
- throws IOException, YarnException;
-
- GetStatusResponseProto getStatus(GetStatusRequestProto requestProto)
- throws IOException, YarnException;
-
- StopResponseProto stop(StopRequestProto requestProto)
- throws IOException, YarnException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
deleted file mode 100644
index 6884757..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
-
-public class ClientAMService extends AbstractService
- implements ClientAMProtocol {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ClientAMService.class);
-
- private ServiceContext context;
- private Server server;
-
- private InetSocketAddress bindAddress;
-
- public ClientAMService(ServiceContext context) {
- super("Client AM Service");
- this.context = context;
- }
-
- @Override protected void serviceStart() throws Exception {
- Configuration conf = getConfig();
- YarnRPC rpc = YarnRPC.create(conf);
- InetSocketAddress address = new InetSocketAddress(0);
- server = rpc.getServer(ClientAMProtocol.class, this, address, conf,
- context.secretManager, 1);
- server.start();
-
- String nodeHostString =
- System.getenv(ApplicationConstants.Environment.NM_HOST.name());
-
- bindAddress = NetUtils.createSocketAddrForHost(nodeHostString,
- server.getListenerAddress().getPort());
-
- LOG.info("Instantiated ClientAMService at " + bindAddress);
- super.serviceStart();
- }
-
- @Override protected void serviceStop() throws Exception {
- if (server != null) {
- server.stop();
- }
- super.serviceStop();
- }
-
- @Override public FlexComponentsResponseProto flexComponents(
- FlexComponentsRequestProto request) throws IOException {
- if (!request.getComponentsList().isEmpty()) {
- for (ComponentCountProto component : request.getComponentsList()) {
- ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
- .setDesired(component.getNumberOfContainers());
- context.scheduler.getDispatcher().getEventHandler().handle(event);
- LOG.info("Flexing component {} to {}", component.getName(),
- component.getNumberOfContainers());
- }
- }
- return FlexComponentsResponseProto.newBuilder().build();
- }
-
- @Override
- public GetStatusResponseProto getStatus(GetStatusRequestProto request)
- throws IOException, YarnException {
- String stat = ServiceApiUtil.jsonSerDeser.toJson(context.application);
- return GetStatusResponseProto.newBuilder().setStatus(stat).build();
- }
-
- @Override
- public StopResponseProto stop(StopRequestProto requestProto)
- throws IOException, YarnException {
- LOG.info("Stop the service.");
- // Stop the service in 2 seconds delay to make sure this rpc call is completed.
- // shutdown hook will be executed which will stop AM gracefully.
- Thread thread = new Thread() {
- @Override
- public void run() {
- try {
- Thread.sleep(2000);
- ExitUtil.terminate(0);
- } catch (InterruptedException e) {
- LOG.error("Interrupted while stopping", e);
- }
- }
- };
- thread.start();
- return StopResponseProto.newBuilder().build();
- }
-
- public InetSocketAddress getBindAddress() {
- return bindAddress;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
deleted file mode 100644
index 4743f28..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.yarn.service.component.Component;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.NODE_BLACKLIST_THRESHOLD;
-
-/**
- * This tracks the container failures per node. If the failure counter exceeds
- * the maxFailurePerNode limit, it'll blacklist that node.
- *
- */
-public class ContainerFailureTracker {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerFailureTracker.class);
-
- // Host -> num container failures
- private Map<String, Integer> failureCountPerNode = new HashMap<>();
- private Set<String> blackListedNodes = new HashSet<>();
- private ServiceContext context;
- private int maxFailurePerNode;
- private Component component;
-
- public ContainerFailureTracker(ServiceContext context, Component component) {
- this.context = context;
- this.component = component;
- maxFailurePerNode = component.getComponentSpec().getConfiguration()
- .getPropertyInt(NODE_BLACKLIST_THRESHOLD, 3);
- }
-
-
- public synchronized void incNodeFailure(String host) {
- int num = 0;
- if (failureCountPerNode.containsKey(host)) {
- num = failureCountPerNode.get(host);
- }
- num++;
- failureCountPerNode.put(host, num);
-
- // black list the node if exceed max failure
- if (num > maxFailurePerNode && !blackListedNodes.contains(host)) {
- List<String> blacklists = new ArrayList<>();
- blacklists.add(host);
- blackListedNodes.add(host);
- context.scheduler.getAmRMClient().updateBlacklist(blacklists, null);
- LOG.info("[COMPONENT {}]: Failed {} times on this host, blacklisted {}."
- + " Current list of blacklisted nodes: {}",
- component.getName(), num, host, blackListedNodes);
- }
- }
-
- public synchronized void resetContainerFailures() {
- // reset container failure counter per node
- failureCountPerNode.clear();
- context.scheduler.getAmRMClient()
- .updateBlacklist(null, new ArrayList<>(blackListedNodes));
- LOG.info("[COMPONENT {}]: Clearing blacklisted nodes {} ",
- component.getName(), blackListedNodes);
- blackListedNodes.clear();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
deleted file mode 100644
index c7616af..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.cache.LoadingCache;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-
-public class ServiceContext {
- public Application application = null;
- public SliderFileSystem fs;
- public String serviceHdfsDir = "";
- public ApplicationAttemptId attemptId;
- public LoadingCache<ConfigFile, Object> configCache;
- public ServiceScheduler scheduler;
- public ClientToAMTokenSecretManager secretManager;
- public ClientAMService clientAMService;
-
- public ServiceContext() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
deleted file mode 100644
index d099f8c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.servicemonitor.ServiceMonitor;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class ServiceMaster extends CompositeService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceMaster.class);
-
- private static SliderAMArgs amArgs;
- protected ServiceContext context;
-
- public ServiceMaster(String name) {
- super(name);
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- //TODO Deprecate slider conf, make sure works with yarn conf
- printSystemEnv();
- if (UserGroupInformation.isSecurityEnabled()) {
- UserGroupInformation.setConfiguration(conf);
- }
- LOG.info("Login user is {}", UserGroupInformation.getLoginUser());
-
- context = new ServiceContext();
- Path appDir = getAppDir();
- context.serviceHdfsDir = appDir.toString();
- SliderFileSystem fs = new SliderFileSystem(conf);
- context.fs = fs;
- fs.setAppDir(appDir);
- loadApplicationJson(context, fs);
-
- ContainerId amContainerId = getAMContainerId();
-
- ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId();
- LOG.info("Application attemptId: " + attemptId);
- context.attemptId = attemptId;
-
- // configure AM to wait forever for RM
- conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
- conf.unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
-
- DefaultMetricsSystem.initialize("ServiceAppMaster");
-
- context.secretManager = new ClientToAMTokenSecretManager(attemptId, null);
- ClientAMService clientAMService = new ClientAMService(context);
- context.clientAMService = clientAMService;
- addService(clientAMService);
-
- ServiceScheduler scheduler = createServiceScheduler(context);
- addService(scheduler);
- context.scheduler = scheduler;
-
- ServiceMonitor monitor = new ServiceMonitor("Service Monitor", context);
- addService(monitor);
-
- super.serviceInit(conf);
- }
-
- protected ContainerId getAMContainerId() throws BadClusterStateException {
- return ContainerId.fromString(SliderUtils.mandatoryEnvVariable(
- ApplicationConstants.Environment.CONTAINER_ID.name()));
- }
-
- protected Path getAppDir() {
- return new Path(amArgs.getAppDefPath()).getParent();
- }
-
- protected ServiceScheduler createServiceScheduler(ServiceContext context)
- throws IOException, YarnException {
- return new ServiceScheduler(context);
- }
-
- protected void loadApplicationJson(ServiceContext context,
- SliderFileSystem fs) throws IOException {
- context.application = ServiceApiUtil
- .loadApplicationFrom(fs, new Path(amArgs.getAppDefPath()));
- LOG.info(context.application.toString());
- }
-
- @Override
- protected void serviceStop() throws Exception {
- LOG.info("Stopping app master");
- super.serviceStop();
- }
-
- private void printSystemEnv() {
- for (Map.Entry<String, String> envs : System.getenv().entrySet()) {
- LOG.info("{} = {}", envs.getKey(), envs.getValue());
- }
- }
-
- public static void main(String[] args) throws Exception {
- Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
- StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
- amArgs = new SliderAMArgs(args);
- amArgs.parse();
- try {
- ServiceMaster serviceMaster = new ServiceMaster("Service Master");
- ShutdownHookManager.get()
- .addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30);
- YarnConfiguration conf = new YarnConfiguration();
- new GenericOptionsParser(conf, args);
- serviceMaster.init(conf);
- serviceMaster.start();
- } catch (Throwable t) {
- LOG.error("Error starting service master", t);
- ExitUtil.terminate(1, "Error starting service master");
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/58] [abbrv] hadoop git commit: YARN-6744. Recover component
information on YARN native services AM restart. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-6744. Recover component information on YARN native services AM restart. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8a7ef1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8a7ef1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8a7ef1b
Branch: refs/heads/yarn-native-services
Commit: b8a7ef1b64392094562e7782e0fd092934724ad2
Parents: c723021
Author: Jian He <ji...@apache.org>
Authored: Wed Oct 11 21:05:06 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:18 2017 -0800
----------------------------------------------------------------------
.../hadoop-yarn-services-core/pom.xml | 5 ++
.../hadoop/yarn/service/ServiceScheduler.java | 94 ++++++++++++++------
.../yarn/service/component/Component.java | 76 +++++++++++++---
.../service/component/ComponentEventType.java | 1 +
.../component/instance/ComponentInstance.java | 27 +++++-
.../yarn/service/provider/ProviderUtils.java | 15 ++--
.../registry/YarnRegistryViewForProviders.java | 52 ++++++++---
7 files changed, 211 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index 205a64d..851f73b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -159,6 +159,11 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index ec5f3ed..f3824df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
@@ -237,11 +237,6 @@ public class ServiceScheduler extends CompositeService {
serviceTimelinePublisher
.serviceAttemptUnregistered(context, diagnostics.toString());
}
- // Cleanup each component instance. no need to release containers as
- // they will be automatically released by RM
- for (ComponentInstance instance : liveInstances.values()) {
- instance.cleanupRegistryAndCompHdfsDir();
- }
String msg = diagnostics.toString()
+ "Navigate to the failed component for more details.";
amRMClient
@@ -266,11 +261,67 @@ public class ServiceScheduler extends CompositeService {
}
registerServiceInstance(context.attemptId, app);
- //TODO handle containers recover
- }
+ // recover components based on containers sent from RM
+ recoverComponents(response);
- private void recover() {
+ for (Component component : componentsById.values()) {
+ // Trigger initial evaluation of components
+ if (component.areDependenciesReady()) {
+ LOG.info("Triggering initial evaluation of component {}",
+ component.getName());
+ ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
+ .setDesired(component.getComponentSpec().getNumberOfContainers());
+ component.handle(event);
+ }
+ }
+ }
+ private void recoverComponents(RegisterApplicationMasterResponse response) {
+ List<Container> recoveredContainers = response
+ .getContainersFromPreviousAttempts();
+ LOG.info("Received {} containers from previous attempt.",
+ recoveredContainers.size());
+ Map<String, ServiceRecord> existingRecords = new HashMap<>();
+ List<String> existingComps = null;
+ try {
+ existingComps = yarnRegistryOperations.listComponents();
+ LOG.info("Found {} containers from ZK registry: {}", existingComps.size(),
+ existingComps);
+ } catch (Exception e) {
+ LOG.info("Could not read component paths: {}", e.getMessage());
+ }
+ if (existingComps != null) {
+ for (String existingComp : existingComps) {
+ try {
+ ServiceRecord record =
+ yarnRegistryOperations.getComponent(existingComp);
+ existingRecords.put(existingComp, record);
+ } catch (Exception e) {
+ LOG.warn("Could not resolve record for component {}: {}",
+ existingComp, e);
+ }
+ }
+ }
+ for (Container container : recoveredContainers) {
+ LOG.info("Handling container {} from previous attempt",
+ container.getId());
+ ServiceRecord record = existingRecords.get(RegistryPathUtils
+ .encodeYarnID(container.getId().toString()));
+ if (record != null) {
+ Component comp = componentsById.get(container.getAllocationRequestId());
+ ComponentEvent event =
+ new ComponentEvent(comp.getName(), CONTAINER_RECOVERED)
+ .setContainer(container)
+ .setInstance(comp.getComponentInstance(record.description));
+ comp.handle(event);
+ // do not remove requests in this case because we do not know if they
+ // have already been removed
+ } else {
+ LOG.info("Record not found in registry for container {} from previous" +
+ " attempt, releasing", container.getId());
+ amRMClient.releaseAssignedContainer(container.getId());
+ }
+ }
}
private void initGlobalTokensForSubstitute(ServiceContext context) {
@@ -353,7 +404,7 @@ public class ServiceScheduler extends CompositeService {
executorService.submit(new Runnable() {
@Override public void run() {
try {
- yarnRegistryOperations.registerSelf(serviceRecord, true);
+ yarnRegistryOperations.registerSelf(serviceRecord, false);
LOG.info("Registered service under {}; absolute path {}",
yarnRegistryOperations.getSelfRegistrationPath(),
yarnRegistryOperations.getAbsoluteSelfRegistrationPath());
@@ -398,13 +449,6 @@ public class ServiceScheduler extends CompositeService {
componentsById.put(allocateId, component);
componentsByName.put(component.getName(), component);
allocateId++;
-
- // Trigger the component without dependencies
- if (component.areDependenciesReady()) {
- ComponentEvent event = new ComponentEvent(compSpec.getName(), FLEX)
- .setDesired(compSpec.getNumberOfContainers());
- component.handle(event);
- }
}
}
@@ -458,17 +502,17 @@ public class ServiceScheduler extends CompositeService {
new ComponentEvent(comp.getName(), CONTAINER_ALLOCATED)
.setContainer(container);
dispatcher.getEventHandler().handle(event);
+ Collection<AMRMClient.ContainerRequest> requests = amRMClient
+ .getMatchingRequests(container.getAllocationRequestId());
LOG.info("[COMPONENT {}]: {} outstanding container requests.",
- comp.getName(),
- amRMClient.getMatchingRequests(container.getAllocationRequestId()).size());
+ comp.getName(), requests.size());
// remove the corresponding request
- Collection<AMRMClient.ContainerRequest> collection = amRMClient
- .getMatchingRequests(container.getAllocationRequestId());
- if (collection.iterator().hasNext()) {
- AMRMClient.ContainerRequest request = collection.iterator().next();
+ if (requests.iterator().hasNext()) {
+ LOG.info("[COMPONENT {}]: removing one container request.", comp
+ .getName());
+ AMRMClient.ContainerRequest request = requests.iterator().next();
amRMClient.removeContainerRequest(request);
}
-
}
}
@@ -478,7 +522,7 @@ public class ServiceScheduler extends CompositeService {
ContainerId containerId = status.getContainerId();
ComponentInstance instance = liveInstances.get(status.getContainerId());
if (instance == null) {
- LOG.error(
+ LOG.warn(
"Container {} Completed. No component instance exists. exitStatus={}. diagnostics={} ",
containerId, status.getExitStatus(), status.getDiagnostics());
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index cbaf472..98bb238 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.service.ServiceContext;
import org.apache.hadoop.yarn.service.ServiceScheduler;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
import org.apache.hadoop.yarn.service.ServiceMetrics;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
@@ -78,7 +79,7 @@ public class Component implements EventHandler<ComponentEvent> {
private ServiceContext context;
private AMRMClientAsync<ContainerRequest> amrmClient;
private AtomicLong instanceIdCounter = new AtomicLong();
- private Map<ComponentInstanceId, ComponentInstance> compInstances =
+ private Map<String, ComponentInstance> compInstances =
new ConcurrentHashMap<>();
// component instances to be assigned with a container
private List<ComponentInstance> pendingInstances = new LinkedList<>();
@@ -101,6 +102,9 @@ public class Component implements EventHandler<ComponentEvent> {
// INIT will only got to FLEXING
.addTransition(INIT, EnumSet.of(STABLE, FLEXING),
FLEX, new FlexComponentTransition())
+ // container recovered on AM restart
+ .addTransition(INIT, INIT, CONTAINER_RECOVERED,
+ new ContainerRecoveredTransition())
// container allocated by RM
.addTransition(FLEXING, FLEXING, CONTAINER_ALLOCATED,
@@ -165,7 +169,7 @@ public class Component implements EventHandler<ComponentEvent> {
new ComponentInstanceId(instanceIdCounter.getAndIncrement(),
componentSpec.getName());
ComponentInstance instance = new ComponentInstance(this, id);
- compInstances.put(id, instance);
+ compInstances.put(instance.getCompInstanceName(), instance);
pendingInstances.add(instance);
}
@@ -186,8 +190,8 @@ public class Component implements EventHandler<ComponentEvent> {
// This happens on init
LOG.info("[INIT COMPONENT " + component.getName() + "]: " + event
.getDesired() + " instances.");
- component.requestContainers(event.getDesired());
- return FLEXING;
+ component.requestContainers(component.pendingInstances.size());
+ return checkIfStable(component);
}
long before = component.getComponentSpec().getNumberOfContainers();
long delta = event.getDesired() - before;
@@ -205,14 +209,14 @@ public class Component implements EventHandler<ComponentEvent> {
LOG.info("[FLEX DOWN COMPONENT " + component.getName()
+ "]: scaling down from " + before + " to " + event.getDesired());
List<ComponentInstance> list =
- new ArrayList<>(component.compInstances.values());
+ new ArrayList<>(component.getAllComponentInstances());
// sort in Most recent -> oldest order, destroy most recent ones.
Collections.sort(list, Collections.reverseOrder());
for (int i = 0; i < delta; i++) {
ComponentInstance instance = list.get(i);
// remove the instance
- component.compInstances.remove(instance.getCompInstanceId());
+ component.compInstances.remove(instance.getCompInstanceName());
component.pendingInstances.remove(instance);
component.componentMetrics.containersFailed.incr();
component.componentMetrics.containersRunning.decr();
@@ -236,6 +240,46 @@ public class Component implements EventHandler<ComponentEvent> {
}
}
+ private static class ContainerRecoveredTransition extends BaseTransition {
+ @Override
+ public void transition(Component component, ComponentEvent event) {
+ ComponentInstance instance = event.getInstance();
+ Container container = event.getContainer();
+ if (instance == null) {
+ LOG.info("[COMPONENT {}]: Trying to recover {} but event did not " +
+ "specify component instance",
+ component.getName(), container.getId());
+ component.releaseContainer(container);
+ return;
+ }
+ if (instance.hasContainer()) {
+ LOG.info(
+ "[COMPONENT {}]: Instance {} already has container, release " +
+ "surplus container {}",
+ instance.getCompName(), instance.getCompInstanceId(), container
+ .getId());
+ component.releaseContainer(container);
+ return;
+ }
+ component.pendingInstances.remove(instance);
+ LOG.info("[COMPONENT {}]: Recovered {} for component instance {} on " +
+ "host {}, num pending component instances reduced to {} ",
+ component.getName(), container.getId(), instance
+ .getCompInstanceName(), container.getNodeId(), component
+ .pendingInstances.size());
+ instance.setContainer(container);
+ ProviderUtils.initCompInstanceDir(component.getContext().fs, instance);
+ component.getScheduler().addLiveCompInstance(container.getId(), instance);
+ LOG.info("[COMPONENT {}]: Marking {} as started for component " +
+ "instance {}", component.getName(), event.getContainer().getId(),
+ instance.getCompInstanceId());
+ component.compInstanceDispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(instance.getContainerId(),
+ START));
+ component.incRunningContainers();
+ }
+ }
+
private static class ContainerStartedTransition implements
MultipleArcTransition<Component,ComponentEvent,ComponentState> {
@@ -280,14 +324,18 @@ public class Component implements EventHandler<ComponentEvent> {
return componentMetrics;
}
+ private void releaseContainer(Container container) {
+ scheduler.getAmRMClient().releaseAssignedContainer(container.getId());
+ componentMetrics.surplusContainers.incr();
+ scheduler.getServiceMetrics().surplusContainers.incr();
+ }
+
private void assignContainerToCompInstance(Container container) {
if (pendingInstances.size() == 0) {
LOG.info(
"[COMPONENT {}]: No pending component instance left, release surplus container {}",
getName(), container.getId());
- scheduler.getAmRMClient().releaseAssignedContainer(container.getId());
- componentMetrics.surplusContainers.incr();
- scheduler.getServiceMetrics().surplusContainers.incr();
+ releaseContainer(container);
return;
}
ComponentInstance instance = pendingInstances.remove(0);
@@ -397,7 +445,7 @@ public class Component implements EventHandler<ComponentEvent> {
}
for (String dependency : dependencies) {
Collection<ComponentInstance> instances = scheduler.getAllComponents()
- .get(dependency).getAllComponentInstances().values();
+ .get(dependency).getAllComponentInstances();
for (ComponentInstance instance : instances) {
if (instance.getContainerStatus() == null) {
continue;
@@ -447,8 +495,12 @@ public class Component implements EventHandler<ComponentEvent> {
return componentMetrics.containersDesired.value();
}
- public Map<ComponentInstanceId, ComponentInstance> getAllComponentInstances() {
- return compInstances;
+ public ComponentInstance getComponentInstance(String componentInstanceName) {
+ return compInstances.get(componentInstanceName);
+ }
+
+ public Collection<ComponentInstance> getAllComponentInstances() {
+ return compInstances.values();
}
public org.apache.hadoop.yarn.service.api.records.Component getComponentSpec() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
index 6729699..067302d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.service.component;
public enum ComponentEventType {
FLEX,
CONTAINER_ALLOCATED,
+ CONTAINER_RECOVERED,
CONTAINER_STARTED,
CONTAINER_COMPLETED
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 3c1e48f..68c0537 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.Container;
@@ -35,6 +34,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.service.ServiceScheduler;
import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.component.Component;
@@ -143,10 +144,19 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
compInstance.getContainerId(), compInstance), 0, 1,
TimeUnit.SECONDS);
+ long containerStartTime = System.currentTimeMillis();
+ try {
+ ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
+ .newContainerTokenIdentifier(compInstance.getContainer()
+ .getContainerToken());
+ containerStartTime = containerTokenIdentifier.getCreationTime();
+ } catch (Exception e) {
+ LOG.info("Could not get container creation time, using current time");
+ }
org.apache.hadoop.yarn.service.api.records.Container container =
new org.apache.hadoop.yarn.service.api.records.Container();
container.setId(compInstance.getContainerId().toString());
- container.setLaunchTime(new Date());
+ container.setLaunchTime(new Date(containerStartTime));
container.setState(ContainerState.RUNNING_BUT_UNREADY);
container.setBareHost(compInstance.container.getNodeId().getHost());
container.setComponentName(compInstance.getCompInstanceName());
@@ -156,7 +166,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
compInstance.containerSpec = container;
compInstance.getCompSpec().addContainer(container);
- compInstance.containerStartedTime = System.currentTimeMillis();
+ compInstance.containerStartedTime = containerStartTime;
if (compInstance.timelineServiceEnabled) {
compInstance.serviceTimelinePublisher
@@ -243,6 +253,8 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
ExitUtil.terminate(-1);
}
+
+ compInstance.removeContainer();
}
}
@@ -276,6 +288,15 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
}
+ public boolean hasContainer() {
+ return this.container != null;
+ }
+
+ public void removeContainer() {
+ this.container = null;
+ this.compInstanceId.setContainerId(null);
+ }
+
public void setContainer(Container container) {
this.container = container;
this.compInstanceId.setContainerId(container.getId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 93abd73..63fbaae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -209,21 +209,26 @@ public class ProviderUtils implements YarnServiceConstants {
}
}
+ public static Path initCompInstanceDir(SliderFileSystem fs,
+ ComponentInstance instance) {
+ Path compDir = new Path(new Path(fs.getAppDir(), "components"),
+ instance.getCompName());
+ Path compInstanceDir = new Path(compDir, instance.getCompInstanceName());
+ instance.setCompInstanceDir(compInstanceDir);
+ return compInstanceDir;
+ }
+
// 1. Create all config files for a component on hdfs for localization
// 2. Add the config file to localResource
public static synchronized void createConfigFileAndAddLocalResource(
AbstractLauncher launcher, SliderFileSystem fs, Component component,
Map<String, String> tokensForSubstitution, ComponentInstance instance,
ServiceContext context) throws IOException {
- Path compDir =
- new Path(new Path(fs.getAppDir(), "components"), component.getName());
- Path compInstanceDir =
- new Path(compDir, instance.getCompInstanceName());
+ Path compInstanceDir = initCompInstanceDir(fs, instance);
if (!fs.getFileSystem().exists(compInstanceDir)) {
log.info(instance.getCompInstanceId() + ": Creating dir on hdfs: " + compInstanceDir);
fs.getFileSystem().mkdirs(compInstanceDir,
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
- instance.setCompInstanceDir(compInstanceDir);
} else {
log.info("Component instance conf dir already exists: " + compInstanceDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
index 62d7a6a..d418b59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -48,7 +48,7 @@ public class YarnRegistryViewForProviders {
private final RegistryOperations registryOperations;
private final String user;
- private final String sliderServiceClass;
+ private final String serviceClass;
private final String instanceName;
/**
* Record used where the service registered itself.
@@ -57,20 +57,20 @@ public class YarnRegistryViewForProviders {
private ServiceRecord selfRegistration;
/**
- * Path where record was registered
+ * Path where record was registered.
* Null until the service is registered
*/
private String selfRegistrationPath;
public YarnRegistryViewForProviders(RegistryOperations registryOperations,
String user,
- String sliderServiceClass,
+ String serviceClass,
String instanceName,
ApplicationAttemptId applicationAttemptId) {
Preconditions.checkArgument(registryOperations != null,
"null registry operations");
Preconditions.checkArgument(user != null, "null user");
- Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
+ Preconditions.checkArgument(SliderUtils.isSet(serviceClass),
"unset service class");
Preconditions.checkArgument(SliderUtils.isSet(instanceName),
"instanceName");
@@ -78,7 +78,7 @@ public class YarnRegistryViewForProviders {
"null applicationAttemptId");
this.registryOperations = registryOperations;
this.user = user;
- this.sliderServiceClass = sliderServiceClass;
+ this.serviceClass = serviceClass;
this.instanceName = instanceName;
}
@@ -117,7 +117,7 @@ public class YarnRegistryViewForProviders {
}
/**
- * Add a component under the slider name/entry
+ * Add a component under the slider name/entry.
* @param componentName component name
* @param record record to put
* @throws IOException
@@ -125,13 +125,13 @@ public class YarnRegistryViewForProviders {
public void putComponent(String componentName,
ServiceRecord record) throws
IOException {
- putComponent(sliderServiceClass, instanceName,
+ putComponent(serviceClass, instanceName,
componentName,
record);
}
/**
- * Add a component
+ * Add a component.
* @param serviceClass service class to use under ~user
* @param componentName component name
* @param record record to put
@@ -146,9 +146,33 @@ public class YarnRegistryViewForProviders {
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.bind(path, record, BindFlags.OVERWRITE);
}
-
+
+ /**
+ * Get a component.
+ * @param componentName component name
+ * @return the service record
+ * @throws IOException
+ */
+ public ServiceRecord getComponent(String componentName) throws IOException {
+ String path = RegistryUtils.componentPath(
+ user, serviceClass, instanceName, componentName);
+ LOG.info("Resolving path {}", path);
+ return registryOperations.resolve(path);
+ }
+
+ /**
+ * List components.
+ * @return a list of components
+ * @throws IOException
+ */
+ public List<String> listComponents() throws IOException {
+ String path = RegistryUtils.componentListPath(
+ user, serviceClass, instanceName);
+ return registryOperations.list(path);
+ }
+
/**
- * Add a service under a path, optionally purging any history
+ * Add a service under a path, optionally purging any history.
* @param username user
* @param serviceClass service class to use under ~user
* @param serviceName name of the service
@@ -173,7 +197,7 @@ public class YarnRegistryViewForProviders {
}
/**
- * Add a service under a path for the current user
+ * Add a service under a path for the current user.
* @param record service record
* @param deleteTreeFirst perform recursive delete of the path first
* @return the path the service was created at
@@ -183,20 +207,20 @@ public class YarnRegistryViewForProviders {
ServiceRecord record,
boolean deleteTreeFirst) throws IOException {
selfRegistrationPath =
- putService(user, sliderServiceClass, instanceName, record, deleteTreeFirst);
+ putService(user, serviceClass, instanceName, record, deleteTreeFirst);
setSelfRegistration(record);
return selfRegistrationPath;
}
/**
- * Delete a component
+ * Delete a component.
* @param containerId component name
* @throws IOException
*/
public void deleteComponent(ComponentInstanceId instanceId,
String containerId) throws IOException {
String path = RegistryUtils.componentPath(
- user, sliderServiceClass, instanceName,
+ user, serviceClass, instanceName,
containerId);
LOG.info(instanceId + ": Deleting registry path " + path);
registryOperations.delete(path, false);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
new file mode 100644
index 0000000..c3a2752
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -0,0 +1,892 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.ServiceMaster;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.client.params.CommonArgs;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
+import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.text.MessageFormat;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
+import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class ServiceClient extends CompositeService
+ implements SliderExitCodes, YarnServiceConstants {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceClient.class);
+ private SliderFileSystem fs;
+ //TODO disable retry so that client / rest API doesn't block?
+ protected YarnClient yarnClient;
+ // Avoid looking up applicationId from fs all the time.
+ private Map<String, ApplicationId> cachedAppIds = new ConcurrentHashMap<>();
+
+ private RegistryOperations registryClient;
+ private CuratorFramework curatorClient;
+ private YarnRPC rpc;
+
+ private static EnumSet<YarnApplicationState> terminatedStates =
+ EnumSet.of(FINISHED, FAILED, KILLED);
+ private static EnumSet<YarnApplicationState> liveStates =
+ EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING);
+ private static EnumSet<YarnApplicationState> preRunningStates =
+ EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED);
+
+ public ServiceClient() {
+ super(ServiceClient.class.getName());
+ }
+
+ @Override protected void serviceInit(Configuration configuration)
+ throws Exception {
+ fs = new SliderFileSystem(configuration);
+ yarnClient = YarnClient.createYarnClient();
+ rpc = YarnRPC.create(configuration);
+ addService(yarnClient);
+ super.serviceInit(configuration);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (registryClient != null) {
+ registryClient.stop();
+ }
+ super.serviceStop();
+ }
+
+ private Service loadAppJsonFromLocalFS(
+ AbstractClusterBuildingActionArgs args) throws IOException {
+ File file = args.getAppDef();
+ Path filePath = new Path(file.getAbsolutePath());
+ LOG.info("Loading app json from: " + filePath);
+ Service service = jsonSerDeser
+ .load(FileSystem.getLocal(getConfig()), filePath);
+ if (args.lifetime > 0) {
+ service.setLifetime(args.lifetime);
+ }
+ service.setName(args.getClusterName());
+ return service;
+ }
+
+ public int actionBuild(AbstractClusterBuildingActionArgs args)
+ throws IOException, YarnException {
+ return actionBuild(loadAppJsonFromLocalFS(args));
+ }
+
+ public int actionBuild(Service service)
+ throws YarnException, IOException {
+ Path appDir = checkAppNotExistOnHdfs(service);
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ createDirAndPersistApp(appDir, service);
+ return EXIT_SUCCESS;
+ }
+
+ public int actionCreate(AbstractClusterBuildingActionArgs args)
+ throws IOException, YarnException {
+ actionCreate(loadAppJsonFromLocalFS(args));
+ return EXIT_SUCCESS;
+ }
+
+ public ApplicationId actionCreate(Service service)
+ throws IOException, YarnException {
+ String serviceName = service.getName();
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ verifyNoLiveAppInRM(serviceName, "create");
+ Path appDir = checkAppNotExistOnHdfs(service);
+
+ // Write the definition first and then submit - AM will read the definition
+ createDirAndPersistApp(appDir, service);
+ ApplicationId appId = submitApp(service);
+ cachedAppIds.put(serviceName, appId);
+ service.setId(appId.toString());
+ // update app definition with appId
+ persistAppDef(appDir, service);
+ return appId;
+ }
+
+ // Called by ServiceCLI
+ protected int actionFlexByCLI(ClientArgs args)
+ throws YarnException, IOException {
+ ActionFlexArgs flexArgs = args.getActionFlexArgs();
+ Map<String, Long> componentCounts =
+ new HashMap<>(flexArgs.getComponentMap().size());
+ Service persistedService =
+ ServiceApiUtil.loadService(fs, flexArgs.getClusterName());
+ if (!StringUtils.isEmpty(persistedService.getId())) {
+ cachedAppIds.put(persistedService.getName(),
+ ApplicationId.fromString(persistedService.getId()));
+ } else {
+ throw new YarnException(persistedService.getName()
+ + " appId is null, may be not submitted to YARN yet");
+ }
+
+ for (Map.Entry<String, String> entry : flexArgs.getComponentMap()
+ .entrySet()) {
+ String compName = entry.getKey();
+ ServiceApiUtil.validateNameFormat(compName, getConfig());
+ Component component = persistedService.getComponent(compName);
+ if (component == null) {
+ throw new IllegalArgumentException(entry.getKey() + " does not exist !");
+ }
+ long numberOfContainers =
+ parseNumberOfContainers(component, entry.getValue());
+ componentCounts.put(compName, numberOfContainers);
+ }
+ // throw usage exception if no changes proposed
+ if (componentCounts.size() == 0) {
+ actionHelp(ACTION_FLEX, args);
+ }
+ flexComponents(args.getClusterName(), componentCounts, persistedService);
+ return EXIT_SUCCESS;
+ }
+
+ // Parse the number of containers requested by user, e.g.
+ // +5 means add 5 additional containers
+ // -5 means reduce 5 containers, if it goes to negative, sets it to 0
+ // 5 means sets it to 5 containers.
+ private long parseNumberOfContainers(Component component, String newNumber) {
+
+ long orig = component.getNumberOfContainers();
+ if (newNumber.startsWith("+")) {
+ return orig + Long.parseLong(newNumber.substring(1));
+ } else if (newNumber.startsWith("-")) {
+ long ret = orig - Long.parseLong(newNumber.substring(1));
+ if (ret < 0) {
+ LOG.warn(MessageFormat.format(
+ "[COMPONENT {}]: component count goes to negative ({}{} = {}), reset it to 0.",
+ component.getName(), orig, newNumber, ret));
+ ret = 0;
+ }
+ return ret;
+ } else {
+ return Long.parseLong(newNumber);
+ }
+ }
+
+ // Called by Rest Service
+ public Map<String, Long> flexByRestService(String serviceName,
+ Map<String, Long> componentCounts) throws YarnException, IOException {
+ // load app definition
+ Service persistedService = ServiceApiUtil.loadService(fs, serviceName);
+ if (StringUtils.isEmpty(persistedService.getId())) {
+ throw new YarnException(
+ serviceName + " appId is null, may be not submitted to YARN yet");
+ }
+ cachedAppIds.put(persistedService.getName(),
+ ApplicationId.fromString(persistedService.getId()));
+ return flexComponents(serviceName, componentCounts, persistedService);
+ }
+
+ private Map<String, Long> flexComponents(String serviceName,
+ Map<String, Long> componentCounts, Service persistedService)
+ throws YarnException, IOException {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+
+ Map<String, Long> original = new HashMap<>(componentCounts.size());
+
+ ComponentCountProto.Builder countBuilder = ComponentCountProto.newBuilder();
+ FlexComponentsRequestProto.Builder requestBuilder =
+ FlexComponentsRequestProto.newBuilder();
+
+ for (Component persistedComp : persistedService.getComponents()) {
+ String name = persistedComp.getName();
+ if (componentCounts.containsKey(persistedComp.getName())) {
+ original.put(name, persistedComp.getNumberOfContainers());
+ persistedComp.setNumberOfContainers(componentCounts.get(name));
+
+ // build the request
+ countBuilder.setName(persistedComp.getName())
+ .setNumberOfContainers(persistedComp.getNumberOfContainers());
+ requestBuilder.addComponents(countBuilder.build());
+ }
+ }
+ if (original.size() < componentCounts.size()) {
+ componentCounts.keySet().removeAll(original.keySet());
+ throw new YarnException("Components " + componentCounts.keySet()
+ + " do not exist in app definition.");
+ }
+ jsonSerDeser
+ .save(fs.getFileSystem(), ServiceApiUtil.getServiceJsonPath(fs, serviceName),
+ persistedService, true);
+
+ ApplicationReport appReport =
+ yarnClient.getApplicationReport(getAppId(serviceName));
+ if (appReport.getYarnApplicationState() != RUNNING) {
+ String message =
+ serviceName + " is at " + appReport.getYarnApplicationState()
+ + " state, flex can only be invoked when service is running";
+ LOG.error(message);
+ throw new YarnException(message);
+ }
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ throw new YarnException(serviceName + " AM hostname is empty");
+ }
+ ClientAMProtocol proxy =
+ createAMProxy(appReport.getHost(), appReport.getRpcPort());
+ proxy.flexComponents(requestBuilder.build());
+ for (Map.Entry<String, Long> entry : original.entrySet()) {
+ LOG.info("[COMPONENT {}]: number of containers changed from {} to {}",
+ entry.getKey(), entry.getValue(),
+ componentCounts.get(entry.getKey()));
+ }
+ return original;
+ }
+
+ public int actionStop(String serviceName, boolean waitForAppStopped)
+ throws YarnException, IOException {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ ApplicationId currentAppId = getAppId(serviceName);
+ ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
+ if (terminatedStates.contains(report.getYarnApplicationState())) {
+ LOG.info("Service {} is already in a terminated state {}", serviceName,
+ report.getYarnApplicationState());
+ return EXIT_SUCCESS;
+ }
+ if (preRunningStates.contains(report.getYarnApplicationState())) {
+ String msg = serviceName + " is at " + report.getYarnApplicationState()
+ + ", forcefully killed by user!";
+ yarnClient.killApplication(currentAppId, msg);
+ LOG.info(msg);
+ return EXIT_SUCCESS;
+ }
+ if (StringUtils.isEmpty(report.getHost())) {
+ throw new YarnException(serviceName + " AM hostname is empty");
+ }
+ LOG.info("Stopping service {}, with appId = {}", serviceName, currentAppId);
+ try {
+ ClientAMProtocol proxy =
+ createAMProxy(report.getHost(), report.getRpcPort());
+ cachedAppIds.remove(serviceName);
+ if (proxy != null) {
+ // try to stop the app gracefully.
+ StopRequestProto request = StopRequestProto.newBuilder().build();
+ proxy.stop(request);
+ LOG.info("Service " + serviceName + " is being gracefully stopped...");
+ } else {
+ yarnClient.killApplication(currentAppId,
+ serviceName + " is forcefully killed by user!");
+ LOG.info("Forcefully kill the service: " + serviceName);
+ return EXIT_SUCCESS;
+ }
+
+ if (!waitForAppStopped) {
+ return EXIT_SUCCESS;
+ }
+ // Wait until the app is killed.
+ long startTime = System.currentTimeMillis();
+ int pollCount = 0;
+ while (true) {
+ Thread.sleep(2000);
+ report = yarnClient.getApplicationReport(currentAppId);
+ if (terminatedStates.contains(report.getYarnApplicationState())) {
+ LOG.info("Service " + serviceName + " is stopped.");
+ break;
+ }
+ // Forcefully kill after 10 seconds.
+ if ((System.currentTimeMillis() - startTime) > 10000) {
+ LOG.info("Stop operation timeout stopping, forcefully kill the app "
+ + serviceName);
+ yarnClient.killApplication(currentAppId,
+ "Forcefully kill the app by user");
+ break;
+ }
+ if (++pollCount % 10 == 0) {
+ LOG.info("Waiting for service " + serviceName + " to be stopped.");
+ }
+ }
+ } catch (IOException | YarnException | InterruptedException e) {
+ LOG.info("Failed to stop " + serviceName
+ + " gracefully, forcefully kill the app.");
+ yarnClient.killApplication(currentAppId, "Forcefully kill the app");
+ }
+ return EXIT_SUCCESS;
+ }
+
+ public int actionDestroy(String serviceName) throws Exception {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ verifyNoLiveAppInRM(serviceName, "Destroy");
+ Path appDir = fs.buildClusterDirPath(serviceName);
+ FileSystem fileSystem = fs.getFileSystem();
+ // remove from the appId cache
+ cachedAppIds.remove(serviceName);
+ if (fileSystem.exists(appDir)) {
+ if (fileSystem.delete(appDir, true)) {
+ LOG.info("Successfully deleted service dir for " + serviceName + ": "
+ + appDir);
+ } else {
+ String message =
+ "Failed to delete service + " + serviceName + " at: " + appDir;
+ LOG.info(message);
+ throw new YarnException(message);
+ }
+ }
+ deleteZKNode(serviceName);
+ String registryPath = ServiceRegistryUtils.registryPathForInstance(serviceName);
+ try {
+ getRegistryClient().delete(registryPath, true);
+ } catch (IOException e) {
+ LOG.warn("Error deleting registry entry {}", registryPath, e);
+ }
+ LOG.info("Destroyed cluster {}", serviceName);
+ return EXIT_SUCCESS;
+ }
+
+ private synchronized RegistryOperations getRegistryClient()
+ throws SliderException, IOException {
+
+ if (registryClient == null) {
+ registryClient =
+ RegistryOperationsFactory.createInstance("ServiceClient", getConfig());
+ registryClient.init(getConfig());
+ registryClient.start();
+ }
+ return registryClient;
+ }
+
+ private void deleteZKNode(String clusterName) throws Exception {
+ CuratorFramework curatorFramework = getCuratorClient();
+ String user = RegistryUtils.currentUser();
+ String zkPath = ServiceRegistryUtils.mkClusterPath(user, clusterName);
+ if (curatorFramework.checkExists().forPath(zkPath) != null) {
+ curatorFramework.delete().deletingChildrenIfNeeded().forPath(zkPath);
+ LOG.info("Deleted zookeeper path: " + zkPath);
+ }
+ }
+
+ private synchronized CuratorFramework getCuratorClient()
+ throws BadConfigException {
+ String registryQuorum =
+ getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+
+ // though if neither is set: trouble
+ if (SliderUtils.isUnset(registryQuorum)) {
+ throw new BadConfigException(
+ "No Zookeeper quorum provided in the" + " configuration property "
+ + RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+ }
+ ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
+
+ if (curatorClient == null) {
+ curatorClient =
+ CuratorFrameworkFactory.builder().connectString(registryQuorum)
+ .sessionTimeoutMs(10000).retryPolicy(new RetryNTimes(5, 2000))
+ .build();
+ curatorClient.start();
+ }
+ return curatorClient;
+ }
+
+ private int actionHelp(String actionName, CommonArgs args)
+ throws YarnException, IOException {
+ throw new UsageException(CommonArgs.usage(args, actionName));
+ }
+
+ private void verifyNoLiveAppInRM(String serviceName, String action)
+ throws IOException, YarnException {
+ Set<String> types = new HashSet<>(1);
+ types.add(YarnServiceConstants.APP_TYPE);
+ Set<String> tags = null;
+ if (serviceName != null) {
+ tags = Collections.singleton(SliderUtils.createNameTag(serviceName));
+ }
+ GetApplicationsRequest request = GetApplicationsRequest.newInstance();
+ request.setApplicationTypes(types);
+ request.setApplicationTags(tags);
+ request.setApplicationStates(liveStates);
+ List<ApplicationReport> reports = yarnClient.getApplications(request);
+ if (!reports.isEmpty()) {
+ throw new YarnException(
+ "Failed to " + action + " service, as " + serviceName
+ + " already exists.");
+ }
+ }
+
+ private ApplicationId submitApp(Service app)
+ throws IOException, YarnException {
+ String serviceName = app.getName();
+ Configuration conf = getConfig();
+ Path appRootDir = fs.buildClusterDirPath(app.getName());
+
+ YarnClientApplication yarnApp = yarnClient.createApplication();
+ ApplicationSubmissionContext submissionContext =
+ yarnApp.getApplicationSubmissionContext();
+ ServiceApiUtil.validateCompResourceSize(
+ yarnApp.getNewApplicationResponse().getMaximumResourceCapability(),
+ app);
+
+ submissionContext.setKeepContainersAcrossApplicationAttempts(true);
+ if (app.getLifetime() > 0) {
+ Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>();
+ appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime());
+ submissionContext.setApplicationTimeouts(appTimeout);
+ }
+ submissionContext.setMaxAppAttempts(conf.getInt(
+ YarnServiceConf.AM_RESTART_MAX, 2));
+
+ Map<String, LocalResource> localResources = new HashMap<>();
+
+ // copy local slideram-log4j.properties to hdfs and add to localResources
+ boolean hasAMLog4j =
+ addAMLog4jResource(serviceName, conf, localResources);
+ // copy jars to hdfs and add to localResources
+ addJarResource(serviceName, localResources);
+ // add keytab if in secure env
+ addKeytabResourceIfSecure(fs, localResources, conf, serviceName);
+ if (LOG.isDebugEnabled()) {
+ printLocalResources(localResources);
+ }
+ Map<String, String> env = addAMEnv(conf);
+
+ // create AM CLI
+ String cmdStr =
+ buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j);
+ submissionContext.setResource(Resource.newInstance(YarnServiceConf
+ .getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM,
+ app.getConfiguration(), conf), 1));
+ String queue = app.getQueue();
+ if (StringUtils.isEmpty(queue)) {
+ queue = conf.get(YARN_QUEUE, "default");
+ }
+ submissionContext.setQueue(queue);
+ submissionContext.setApplicationName(serviceName);
+ submissionContext.setApplicationType(YarnServiceConstants.APP_TYPE);
+ Set<String> appTags =
+ AbstractClientProvider.createApplicationTags(serviceName, null, null);
+ if (!appTags.isEmpty()) {
+ submissionContext.setApplicationTags(appTags);
+ }
+ ContainerLaunchContext amLaunchContext =
+ Records.newRecord(ContainerLaunchContext.class);
+ amLaunchContext.setCommands(Collections.singletonList(cmdStr));
+ amLaunchContext.setEnvironment(env);
+ amLaunchContext.setLocalResources(localResources);
+ submissionContext.setAMContainerSpec(amLaunchContext);
+ yarnClient.submitApplication(submissionContext);
+ return submissionContext.getApplicationId();
+ }
+
+ private void printLocalResources(Map<String, LocalResource> map) {
+ LOG.debug("Added LocalResource for localization: ");
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, LocalResource> entry : map.entrySet()) {
+ builder.append(entry.getKey()).append(" -> ")
+ .append(entry.getValue().getResource().getFile())
+ .append(System.lineSeparator());
+ }
+ LOG.debug(builder.toString());
+ }
+
+ private String buildCommandLine(String serviceName, Configuration conf,
+ Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException {
+ JavaCommandLineBuilder CLI = new JavaCommandLineBuilder();
+ CLI.forceIPv4().headless();
+ //TODO CLI.setJVMHeap
+ //TODO CLI.addJVMOPTS
+ if (hasSliderAMLog4j) {
+ CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME);
+ CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ }
+ CLI.add(ServiceMaster.class.getCanonicalName());
+ CLI.add(ACTION_CREATE, serviceName);
+ //TODO debugAM CLI.add(Arguments.ARG_DEBUG)
+ CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, serviceName + ".json"));
+ // pass the registry binding
+ CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
+ CLI.addMandatoryConfOption(conf, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+
+ // write out the path output
+ CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
+ String cmdStr = CLI.build();
+ LOG.info("AM launch command: {}", cmdStr);
+ return cmdStr;
+ }
+
+ private Map<String, String> addAMEnv(Configuration conf) throws IOException {
+ Map<String, String> env = new HashMap<>();
+ ClasspathConstructor classpath =
+ buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
+ .getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
+ env.put("CLASSPATH", classpath.buildClasspath());
+ env.put("LANG", "en_US.UTF-8");
+ env.put("LC_ALL", "en_US.UTF-8");
+ env.put("LANGUAGE", "en_US.UTF-8");
+ String jaas = System.getenv("HADOOP_JAAS_DEBUG");
+ if (jaas != null) {
+ env.put("HADOOP_JAAS_DEBUG", jaas);
+ }
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ String userName = UserGroupInformation.getCurrentUser().getUserName();
+ LOG.info("Run as user " + userName);
+ // HADOOP_USER_NAME env is used by UserGroupInformation when log in
+ // This env makes AM run as this user
+ env.put("HADOOP_USER_NAME", userName);
+ }
+ LOG.info("AM env: \n{}", stringifyMap(env));
+ return env;
+ }
+
+ protected Path addJarResource(String serviceName,
+ Map<String, LocalResource> localResources)
+ throws IOException, SliderException {
+ Path libPath = fs.buildClusterDirPath(serviceName);
+ ProviderUtils
+ .addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
+ libPath, "lib", false);
+ Path dependencyLibTarGzip = fs.getDependencyTarGzip();
+ if (fs.isFile(dependencyLibTarGzip)) {
+ LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ + dependencyLibTarGzip);
+ SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
+ } else {
+ String[] libs = SliderUtils.getLibDirs();
+ for (String libDirProp : libs) {
+ ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
+ libDirProp);
+ }
+ }
+ return libPath;
+ }
+
+ private boolean addAMLog4jResource(String serviceName, Configuration conf,
+ Map<String, LocalResource> localResources)
+ throws IOException, BadClusterStateException {
+ boolean hasAMLog4j = false;
+ String hadoopConfDir =
+ System.getenv(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
+ if (hadoopConfDir != null) {
+ File localFile =
+ new File(hadoopConfDir, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+ if (localFile.exists()) {
+ Path localFilePath = createLocalPath(localFile);
+ Path appDirPath = fs.buildClusterDirPath(serviceName);
+ Path remoteConfPath =
+ new Path(appDirPath, YarnServiceConstants.SUBMITTED_CONF_DIR);
+ Path remoteFilePath =
+ new Path(remoteConfPath, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+ copy(conf, localFilePath, remoteFilePath);
+ LocalResource localResource =
+ fs.createAmResource(remoteConfPath, LocalResourceType.FILE);
+ localResources.put(localFilePath.getName(), localResource);
+ hasAMLog4j = true;
+ } else {
+ LOG.warn("AM log4j property file doesn't exist: " + localFile);
+ }
+ }
+ return hasAMLog4j;
+ }
+
+ public int actionStart(String serviceName) throws YarnException, IOException {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ Path appDir = checkAppExistOnHdfs(serviceName);
+ Service service = ServiceApiUtil.loadService(fs, serviceName);
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ // see if it is actually running and bail out;
+ verifyNoLiveAppInRM(serviceName, "thaw");
+ ApplicationId appId = submitApp(service);
+ service.setId(appId.toString());
+ // write app definition on to hdfs
+ createDirAndPersistApp(appDir, service);
+ return 0;
+ }
+
+ private Path checkAppNotExistOnHdfs(Service service)
+ throws IOException, SliderException {
+ Path appDir = fs.buildClusterDirPath(service.getName());
+ fs.verifyDirectoryNonexistent(
+ new Path(appDir, service.getName() + ".json"));
+ return appDir;
+ }
+
+ private Path checkAppExistOnHdfs(String serviceName)
+ throws IOException, SliderException {
+ Path appDir = fs.buildClusterDirPath(serviceName);
+ fs.verifyPathExists(new Path(appDir, serviceName + ".json"));
+ return appDir;
+ }
+
+ private void createDirAndPersistApp(Path appDir, Service service)
+ throws IOException, SliderException {
+ FsPermission appDirPermission = new FsPermission("750");
+ fs.createWithPermissions(appDir, appDirPermission);
+ persistAppDef(appDir, service);
+ }
+
+ private void persistAppDef(Path appDir, Service service)
+ throws IOException {
+ Path appJson = new Path(appDir, service.getName() + ".json");
+ jsonSerDeser
+ .save(fs.getFileSystem(), appJson, service, true);
+ LOG.info(
+ "Persisted service " + service.getName() + " at " + appJson);
+ }
+
+ private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
+ Map<String, LocalResource> localResource, Configuration conf,
+ String serviceName) throws IOException, BadConfigException {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return;
+ }
+ String keytabPreInstalledOnHost =
+ conf.get(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
+ if (StringUtils.isEmpty(keytabPreInstalledOnHost)) {
+ String amKeytabName =
+ conf.get(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir = conf.get(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
+ Path keytabPath =
+ fileSystem.buildKeytabPath(keytabDir, amKeytabName, serviceName);
+ if (fileSystem.getFileSystem().exists(keytabPath)) {
+ LocalResource keytabRes =
+ fileSystem.createAmResource(keytabPath, LocalResourceType.FILE);
+ localResource
+ .put(YarnServiceConstants.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
+ LOG.info("Adding AM keytab on hdfs: " + keytabPath);
+ } else {
+ LOG.warn("No keytab file was found at {}.", keytabPath);
+ if (conf.getBoolean(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
+ throw new BadConfigException("No keytab file was found at %s.",
+ keytabPath);
+ } else {
+ LOG.warn("The AM will be "
+ + "started without a kerberos authenticated identity. "
+ + "The service is therefore not guaranteed to remain "
+ + "operational beyond 24 hours.");
+ }
+ }
+ }
+ }
+
+ public String updateLifetime(String serviceName, long lifetime)
+ throws YarnException, IOException {
+ ApplicationId currentAppId = getAppId(serviceName);
+ ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
+ if (report == null) {
+ throw new YarnException("Service not found for " + serviceName);
+ }
+ ApplicationId appId = report.getApplicationId();
+ LOG.info("Updating lifetime of an service: serviceName = " + serviceName
+ + ", appId = " + appId + ", lifetime = " + lifetime);
+ Map<ApplicationTimeoutType, String> map = new HashMap<>();
+ String newTimeout =
+ Times.formatISO8601(System.currentTimeMillis() + lifetime * 1000);
+ map.put(ApplicationTimeoutType.LIFETIME, newTimeout);
+ UpdateApplicationTimeoutsRequest request =
+ UpdateApplicationTimeoutsRequest.newInstance(appId, map);
+ yarnClient.updateApplicationTimeouts(request);
+ LOG.info(
+ "Successfully updated lifetime for an service: serviceName = " + serviceName
+ + ", appId = " + appId + ". New expiry time in ISO8601 format is "
+ + newTimeout);
+ return newTimeout;
+ }
+
+ public ServiceState convertState(FinalApplicationStatus status) {
+ switch (status) {
+ case UNDEFINED:
+ return ServiceState.ACCEPTED;
+ case FAILED:
+ case KILLED:
+ return ServiceState.FAILED;
+ case ENDED:
+ case SUCCEEDED:
+ return ServiceState.STOPPED;
+ }
+ return ServiceState.ACCEPTED;
+ }
+
+ public Service getStatus(String serviceName)
+ throws IOException, YarnException {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ ApplicationId currentAppId = getAppId(serviceName);
+ ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId);
+ Service appSpec = new Service();
+ appSpec.setName(serviceName);
+ appSpec.setState(convertState(appReport.getFinalApplicationStatus()));
+ ApplicationTimeout lifetime =
+ appReport.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME);
+ if (lifetime != null) {
+ appSpec.setLifetime(lifetime.getRemainingTime());
+ }
+
+ if (appReport.getYarnApplicationState() != RUNNING) {
+ LOG.info("Service {} is at {} state", serviceName,
+ appReport.getYarnApplicationState());
+ return appSpec;
+ }
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ LOG.warn(serviceName + " AM hostname is empty");
+ return appSpec;
+ }
+ ClientAMProtocol amProxy =
+ createAMProxy(appReport.getHost(), appReport.getRpcPort());
+ GetStatusResponseProto response =
+ amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
+ appSpec = jsonSerDeser.fromJson(response.getStatus());
+
+ return appSpec;
+ }
+
+ public YarnClient getYarnClient() {
+ return this.yarnClient;
+ }
+
+ public int actionDependency(ActionDependencyArgs args)
+ throws IOException, YarnException {
+ String currentUser = RegistryUtils.currentUser();
+ LOG.info("Running command as user {}", currentUser);
+
+ Path dependencyLibTarGzip = fs.getDependencyTarGzip();
+
+ // Check if dependency has already been uploaded, in which case log
+ // appropriately and exit success (unless overwrite has been requested)
+ if (fs.isFile(dependencyLibTarGzip) && !args.overwrite) {
+ System.out.println(String.format(
+ "Dependency libs are already uploaded to %s. Use %s "
+ + "if you want to re-upload", dependencyLibTarGzip.toUri(),
+ Arguments.ARG_OVERWRITE));
+ return EXIT_SUCCESS;
+ }
+
+ String[] libDirs = SliderUtils.getLibDirs();
+ if (libDirs.length > 0) {
+ File tempLibTarGzipFile = File.createTempFile(
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ // copy all jars
+ tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
+
+ LOG.info("Version Info: " + VersionInfo.getBuildVersion());
+ fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
+ new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
+ return EXIT_SUCCESS;
+ } else {
+ return EXIT_FALSE;
+ }
+ }
+
+ protected ClientAMProtocol createAMProxy(String host, int port)
+ throws IOException {
+ InetSocketAddress address =
+ NetUtils.createSocketAddrForHost(host, port);
+ return ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
+ UserGroupInformation.getCurrentUser(), rpc, address);
+ }
+
+ private synchronized ApplicationId getAppId(String serviceName)
+ throws IOException, YarnException {
+ if (cachedAppIds.containsKey(serviceName)) {
+ return cachedAppIds.get(serviceName);
+ }
+ Service persistedService = ServiceApiUtil.loadService(fs, serviceName);
+ if (persistedService == null) {
+ throw new YarnException("Service " + serviceName
+ + " doesn't exist on hdfs. Please check if the app exists in RM");
+ }
+ ApplicationId currentAppId = ApplicationId.fromString(persistedService.getId());
+ cachedAppIds.put(serviceName, currentAppId);
+ return currentAppId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
new file mode 100644
index 0000000..ea3bb0a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Base args for all actions
+ */
+public abstract class AbstractActionArgs extends ArgOps implements Arguments {
+ protected static final Logger log =
+ LoggerFactory.getLogger(AbstractActionArgs.class);
+
+
+ protected AbstractActionArgs() {
+ }
+
+ /**
+ * URI/binding to the filesystem
+ */
+ @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
+ description = "Filesystem Binding")
+ public String filesystemBinding;
+
+ @Parameter(names = {ARG_BASE_PATH},
+ description = "Service base path on the filesystem",
+ converter = PathArgumentConverter.class)
+ public Path basePath;
+
+ /**
+ * This is the default parameter
+ */
+ @Parameter
+ public final List<String> parameters = new ArrayList<>();
+
+ /**
+ * get the name: relies on arg 1 being the cluster name in all operations
+ * @return the name argument, null if there is none
+ */
+ public String getClusterName() {
+ return (parameters.isEmpty()) ? null : parameters.get(0);
+ }
+
+ /**
+ -D name=value
+
+ Define an configuration option which overrides any options in
+ the configuration XML files of the image or in the image configuration
+ directory. The values will be persisted.
+ Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
+
+ */
+
+ @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
+ public final List<String> definitions = new ArrayList<>();
+
+ /**
+ * System properties
+ */
+ @Parameter(names = {ARG_SYSPROP}, arity = 1,
+ description = "system properties in the form name value" +
+ " These are set after the JVM is started.")
+ public final List<String> sysprops = new ArrayList<>(0);
+
+
+ @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
+ description = "Binding (usually hostname:port) of the YARN resource manager")
+ public String manager;
+
+
+ @Parameter(names = ARG_DEBUG, description = "Debug mode")
+ public boolean debug = false;
+
+ @Parameter(names = {ARG_HELP}, description = "Help", help = true)
+ public boolean help = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 1;
+ }
+
+ /**
+ * Get the name of the action
+ * @return the action name
+ */
+ public abstract String getActionName() ;
+
+ /**
+ * Get the max #of params expected
+ * @return the number of params in the {@link #parameters} field;
+ */
+ public int getMaxParams() {
+ return getMinParams();
+ }
+
+ public void validate() throws BadCommandArgumentsException, UsageException {
+
+ int minArgs = getMinParams();
+ int actionArgSize = parameters.size();
+ if (minArgs > actionArgSize) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
+ ", Expected minimum " + minArgs + " but got " + actionArgSize);
+ }
+ int maxArgs = getMaxParams();
+ if (maxArgs == -1) {
+ maxArgs = minArgs;
+ }
+ if (actionArgSize > maxArgs) {
+ String message = String.format("%s for action %s: limit is %d but saw %d: ",
+ ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
+ getActionName(), maxArgs,
+ actionArgSize);
+
+ log.error(message);
+ int index = 1;
+ StringBuilder buf = new StringBuilder(message);
+ for (String actionArg : parameters) {
+ log.error("[{}] \"{}\"", index++, actionArg);
+ buf.append(" \"").append(actionArg).append("\" ");
+ }
+ throw new BadCommandArgumentsException(buf.toString());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + ": " + getActionName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
new file mode 100644
index 0000000..457e357
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import org.apache.hadoop.yarn.service.client.params.ArgOps;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+
+/**
+ * Base class for all the delegates
+ */
+public class AbstractArgsDelegate extends ArgOps implements Arguments {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
new file mode 100644
index 0000000..3a3a19a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract Action to build things; shares args across build and
+ * list
+ */
+public abstract class AbstractClusterBuildingActionArgs
+ extends AbstractActionArgs {
+ @Parameter(names = {ARG_APPDEF},
+ description = "Template service definition file in JSON format.")
+ public File appDef;
+
+ public File getAppDef() {
+ return appDef;
+ }
+
+ @Parameter(names = {
+ ARG_QUEUE }, description = "Queue to submit the service")
+ public String queue;
+
+ @Parameter(names = {
+ ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
+ public long lifetime;
+
+ @ParametersDelegate
+ public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+
+ @ParametersDelegate
+ public OptionArgsDelegate optionsDelegate =
+ new OptionArgsDelegate();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
new file mode 100644
index 0000000..c2ff545
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_BUILD},
+ commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
+
+public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_BUILD;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
new file mode 100644
index 0000000..c43d61a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_CLIENT},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
+
+public class ActionClientArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CLIENT;
+ }
+
+ @Parameter(names = {ARG_INSTALL},
+ description = "Install client")
+ public boolean install;
+
+ @Parameter(names = {ARG_NAME},
+ description = "The name of the service")
+ public String name;
+
+ @Parameter(names = {ARG_PACKAGE},
+ description = "Path to app package")
+ public String packageURI;
+
+ @Parameter(names = {ARG_DEST},
+ description = "The location where to install the client")
+ public File installLocation;
+
+ @Parameter(names = {ARG_CONFIG},
+ description = "Client configuration")
+ public File clientConfig;
+
+ /**
+ * Get the min #of params expected
+ *
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
new file mode 100644
index 0000000..eecffb6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_CREATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CREATE;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
new file mode 100644
index 0000000..51e07c9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client.params;
+
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY },
+ commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY)
+public class ActionDependencyArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_DEPENDENCY;
+ }
+
+ @Parameter(names = { ARG_UPLOAD },
+ description = "Upload AM and agent libraries to HDFS for this client")
+ public boolean upload;
+
+ @Parameter(names = { ARG_OVERWRITE },
+ description = "Overwrite current uploaded dependency libs")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ *
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+
+ if (!upload) {
+ throw new UsageException("Option " + ARG_UPLOAD + " is mandatory");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
new file mode 100644
index 0000000..8c41c04
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_DESTROY},
+ commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
+
+public class ActionDestroyArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_DESTROY;
+ }
+
+ @Parameter(names = {ARG_FORCE},
+ description = "force the operation")
+ public boolean force;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
new file mode 100644
index 0000000..088ad47
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_EXISTS},
+ commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
+
+public class ActionExistsArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_EXISTS;
+ }
+
+ @Parameter(names = {ARG_LIVE},
+ description = "verify that the service is running")
+ public boolean live;
+
+ @Parameter(names = {ARG_STATE},
+ description = "verify that the service is in the specific YARN state")
+ public String state = "";
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any service report")
+ public File out;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
new file mode 100644
index 0000000..b7acf58
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.util.List;
+import java.util.Map;
+
+@Parameters(commandNames = { SliderActions.ACTION_FLEX},
+ commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
+
+public class ActionFlexArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_FLEX;
+ }
+
+ @ParametersDelegate
+ public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+
+ /**
+ * Get the component mapping (may be empty, but never null)
+ * @return mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws
+ BadCommandArgumentsException {
+ return componentDelegate.getComponentMap();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
new file mode 100644
index 0000000..aecf0eb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+@Parameters(commandNames = { SliderActions.ACTION_STOP },
+ commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
+
+public class ActionFreezeArgs extends AbstractActionArgs implements
+ WaitTimeAccessor {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_STOP;
+ }
+
+ public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
+ @ParametersDelegate
+ public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
+
+ @Override
+ public int getWaittime() {
+ return waitDelegate.getWaittime();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ waitDelegate.setWaittime(waittime);
+ }
+
+ @Parameter(names={ARG_MESSAGE},
+ description = "reason for the operation")
+ public String message = FREEZE_COMMAND_ISSUED;
+
+ @Parameter(names = {ARG_FORCE},
+ description = "force the operation")
+ public boolean force;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
new file mode 100644
index 0000000..51aa88a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+/**
+ * The Help command
+ */
+@Parameters(commandNames = { SliderActions.ACTION_HELP},
+ commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
+public class ActionHelpArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_HELP;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
new file mode 100644
index 0000000..061121e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKDiagArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+@Parameters(commandNames = { SliderActions.ACTION_KDIAG},
+ commandDescription = SliderActions.DESCRIBE_ACTION_KDIAG)
+
+public class ActionKDiagArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_KDIAG;
+ }
+
+ @Parameter(names = {ARG_SERVICES}, variableArity = true,
+ description =" list of services to check")
+ public List<String> services = new ArrayList<>();
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for report")
+ public File out;
+
+ @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
+ public File keytab;
+
+ @Parameter(names = {ARG_KEYLEN}, description = "minimum key length")
+ public int keylen = 256;
+
+ @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
+ public String principal;
+
+ @Parameter(names = {ARG_SECURE}, description = "Is security required")
+ public boolean secure = false;
+
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (keytab != null && SliderUtils.isUnset(principal)) {
+ throw new UsageException("Missing argument " + ARG_PRINCIPAL);
+ }
+ if (keytab == null && SliderUtils.isSet(principal)) {
+ throw new UsageException("Missing argument " + ARG_KEYTAB);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
new file mode 100644
index 0000000..7e51457
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_KEYTAB},
+ commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
+
+public class ActionKeytabArgs extends AbstractActionArgs {
+
+ public ActionKeytabArgs() {
+ super();
+ }
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_INSTALL_KEYTAB;
+ }
+
+ @Parameter(names = {ARG_KEYTABINSTALL},
+ description = "Install the keytab")
+ public boolean install;
+
+ @Parameter(names = {ARG_KEYTABDELETE},
+ description = "Delete the keytab")
+ public boolean delete;
+
+ @Parameter(names = {ARG_KEYTABLIST},
+ description = "List of installed keytabs")
+ public boolean list;
+
+ @Parameter(names = {ARG_KEYTAB},
+ description = "Path or name of the keytab")
+ public String keytab;
+
+ @Parameter(names = {ARG_FOLDER},
+ description = "The name of the folder in which to store the keytab")
+ public String folder;
+
+ @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 3;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
new file mode 100644
index 0000000..c05e602
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_LIST},
+ commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
+
+public class ActionListArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_LIST;
+ }
+
+ @Parameter(names = {ARG_LIVE},
+ description = "List only live service instances")
+ public boolean live;
+
+ @Parameter(names = {ARG_STATE},
+ description = "list only applications in the specific YARN state")
+ public String state = "";
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "print out information in details")
+ public boolean verbose = false;
+
+ @Parameter(names = {ARG_CONTAINERS},
+ description = "List containers of a service instance")
+ public boolean containers;
+
+ @Parameter(names = {ARG_VERSION},
+ description = "Filter containers by app version (used with " +
+ ARG_CONTAINERS + ")")
+ public String version;
+
+ @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
+ description = "Filter containers by component names (used with " +
+ ARG_CONTAINERS + ")")
+ public Set<String> components = new HashSet<>(0);
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
new file mode 100644
index 0000000..3e53418
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+import java.io.File;
+
+/**
+ * Registry actions
+ *
+ * --instance {app name}, if a / is in it, refers underneath?
+ * --dest {destfile}
+ * --list : list instances of slider service
+ * --listfiles
+ */
+@Parameters(commandNames = {ACTION_REGISTRY},
+ commandDescription = DESCRIBE_ACTION_REGISTRY)
+
+public class ActionRegistryArgs extends AbstractActionArgs {
+
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_REGISTRY
+ + " ("
+ + Arguments.ARG_LIST + "|"
+ + Arguments.ARG_LISTCONF + "|"
+ + Arguments.ARG_LISTEXP + "|"
+ + Arguments.ARG_LISTFILES + "|"
+ + Arguments.ARG_GETCONF + "|"
+ + Arguments.ARG_GETEXP + "> "
+ + Arguments.ARG_NAME + " <name> "
+ + " )"
+ + "[" + Arguments.ARG_VERBOSE + "] "
+ + "[" + Arguments.ARG_USER + "] "
+ + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
+ + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
+ + System.getProperty("line.separator")
+ + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
+ ;
+ public ActionRegistryArgs() {
+ }
+
+ public ActionRegistryArgs(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getActionName() {
+ return ACTION_REGISTRY;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_LIST},
+ description = "list services")
+ public boolean list;
+
+ @Parameter(names = {ARG_LISTCONF},
+ description = "list configurations")
+ public boolean listConf;
+
+ @Parameter(names = {ARG_GETCONF},
+ description = "get configuration")
+ public String getConf;
+
+ @Parameter(names = {ARG_LISTEXP},
+ description = "list exports")
+ public boolean listExports;
+
+ @Parameter(names = {ARG_GETEXP},
+ description = "get export")
+ public String getExport;
+
+ @Parameter(names = {ARG_LISTFILES},
+ description = "list files")
+ public String listFiles;
+
+ @Parameter(names = {ARG_GETFILES},
+ description = "get files")
+ public String getFiles;
+
+ //--format
+ @Parameter(names = ARG_FORMAT,
+ description = "Format for a response: <xml|json|properties>")
+ public String format = ConfigFormat.XML.toString() ;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
+ description = "Output destination")
+ public File out;
+
+ @Parameter(names = {ARG_NAME},
+ description = "name of an instance")
+ public String name;
+
+ @Parameter(names = {ARG_SERVICETYPE},
+ description = "optional service type")
+ public String serviceType = YarnServiceConstants.APP_TYPE;
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "verbose output")
+ public boolean verbose;
+
+ @Parameter(names = {ARG_INTERNAL},
+ description = "fetch internal registry entries")
+ public boolean internal;
+
+ @Parameter(names = {ARG_USER},
+ description = "the name of the user whose service is being resolved")
+ public String user;
+
+ /**
+ * validate health of all the different operations
+ * @throws BadCommandArgumentsException
+ */
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+
+ //verify that at most one of the operations is set
+ int gets = s(getConf) + s(getFiles) + s(getExport);
+ int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
+ int set = lists + gets;
+ if (set > 1) {
+ throw new UsageException(USAGE);
+ }
+
+ if (out != null && ( set == 0)) {
+ throw new UsageException("output path"
+ + " is only supported on 'get' operations: ");
+ }
+ if (!list && !is(name)) {
+ throw new UsageException("Argument " + ARG_NAME
+ +" missing: ");
+
+ }
+ }
+
+ private int s(String arg) {
+ return is(arg) ? 1 : 0;
+ }
+
+ private boolean is(String arg) {
+ return arg != null;
+ }
+
+ private int s(boolean arg) {
+ return arg ? 1 : 0;
+ }
+
+ private String ifdef(String arg, boolean val) {
+ return val ? (arg + " "): "";
+ }
+
+ private String ifdef(String arg, String val) {
+ if (is(val)) {
+ return arg + " " + val + " ";
+ } else {
+ return "";
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(ACTION_REGISTRY);
+ sb.append(' ');
+ sb.append(ifdef(ARG_LIST, list));
+ sb.append(ifdef(ARG_LISTCONF, listConf));
+ sb.append(ifdef(ARG_LISTFILES, listFiles));
+ sb.append(ifdef(ARG_GETCONF, getConf));
+ sb.append(ifdef(ARG_GETFILES, getFiles));
+
+ sb.append(ifdef(ARG_NAME, name));
+ sb.append(ifdef(ARG_SERVICETYPE, serviceType));
+
+
+ sb.append(ifdef(ARG_VERBOSE, verbose));
+ sb.append(ifdef(ARG_INTERNAL, internal));
+
+ if (out != null) {
+ sb.append(ifdef(ARG_OUTPUT, out.toString()));
+ }
+ sb.append(ifdef(ARG_FORMAT, format));
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
new file mode 100644
index 0000000..65f0472
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import java.io.File;
+
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
+import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+
+/**
+ * Resolve registry entries
+ *
+ * --path {path}
+ * --out {destfile}
+ * --verbose
+ * --list
+ */
+@Parameters(commandNames = {ACTION_RESOLVE},
+ commandDescription = DESCRIBE_ACTION_REGISTRY)
+public class ActionResolveArgs extends AbstractActionArgs {
+
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_RESOLVE
+ + " "
+ + ARG_PATH + " <path> "
+ + "[" + ARG_LIST + "] "
+ + "[" + ARG_OUTPUT + " <filename> ] "
+ + "[" + ARG_DESTDIR + " <directory> ] "
+ ;
+ public ActionResolveArgs() {
+ }
+
+ @Override
+ public String getActionName() {
+ return ACTION_RESOLVE;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_LIST},
+ description = "list services")
+ public boolean list;
+
+ @Parameter(names = {ARG_PATH},
+ description = "resolve a path")
+ public String path;
+
+ @Parameter(names = {ARG_DESTDIR},
+ description = "destination directory for operations")
+ public File destdir;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "dest file")
+ public File out;
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(ACTION_RESOLVE).append(" ");
+ sb.append(ARG_PATH).append(" ").append(path).append(" ");
+ if (list) {
+ sb.append(ARG_LIST).append(" ");
+ }
+ if (destdir != null) {
+ sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
+ }
+ if (out != null) {
+ sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (StringUtils.isEmpty(path)) {
+ throw new BadCommandArgumentsException("Missing mandatory argument "
+ + ARG_PATH);
+ }
+ if (list && out != null) {
+ throw new BadCommandArgumentsException("Argument "
+ + ARG_OUTPUT +
+ " not supported for " + ARG_LIST);
+ }
+ if (out != null && destdir != null) {
+ throw new BadCommandArgumentsException(
+ ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
+ );
+ }
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public boolean isList() {
+ return list;
+ }
+
+ public void setList(boolean list) {
+ this.list = list;
+ }
+
+ public File getDestdir() {
+ return destdir;
+ }
+
+ public void setDestdir(File destdir) {
+ this.destdir = destdir;
+ }
+
+ public File getOut() {
+ return out;
+ }
+
+ public void setOut(File out) {
+ this.out = out;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
new file mode 100644
index 0000000..b03dc92
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_RESOURCE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
+
+public class ActionResourceArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_RESOURCE;
+ }
+
+ @Parameter(names = {ARG_INSTALL},
+ description = "Install the resource(s)")
+ public boolean install;
+
+ @Parameter(names = {ARG_DELETE},
+ description = "Delete the file")
+ public boolean delete;
+
+ @Parameter(names = {ARG_LIST},
+ description = "List of installed files")
+ public boolean list;
+
+ @Parameter(names = {ARG_RESOURCE},
+ description = "Name of the file or directory")
+ public String resource;
+
+ @Parameter(names = {ARG_DESTDIR},
+ description = "The name of the folder in which to store the resources")
+ public String folder;
+
+ @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
new file mode 100644
index 0000000..31f25ef
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+
+@Parameters(commandNames = { SliderActions.ACTION_STATUS},
+ commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
+
+public class ActionStatusArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_STATUS;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "Output file for the status information")
+ public String output;
+
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Lifetime of the service from the time of request")
+ public boolean lifetime;
+
+ public String getOutput() {
+ return output;
+ }
+
+ public void setOutput(String output) {
+ this.output = output;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
new file mode 100644
index 0000000..175e367
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_START },
+ commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
+public class ActionThawArgs extends AbstractActionArgs implements
+ WaitTimeAccessor,
+ LaunchArgsAccessor {
+
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_START;
+ }
+
+ @Override
+ public int getWaittime() {
+ return launchArgs.getWaittime();
+ }
+
+ @ParametersDelegate
+ LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Life time of the service since service started at"
+ + " running state")
+ public long lifetime;
+
+ @Override
+ public String getRmAddress() {
+ return launchArgs.getRmAddress();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ launchArgs.setWaittime(waittime);
+ }
+
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
new file mode 100644
index 0000000..cf48513
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+
+import java.io.File;
+
+@Parameters(commandNames = { SliderActions.ACTION_TOKENS},
+ commandDescription = "save tokens to a file or list tokens in a file")
+public class ActionTokensArgs extends AbstractActionArgs {
+
+ public static final String DUPLICATE_ARGS = "Only one of " +
+ ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
+
+ public static final String MISSING_KT_PROVIDER =
+ "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
+ + " must be provided";
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_TOKENS;
+ }
+
+ @Parameter(names = {ARG_OUTPUT},
+ description = "File to write")
+ public File output;
+
+ @Parameter(names = {ARG_SOURCE},
+ description = "source file")
+ public File source;
+
+ @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
+ public File keytab;
+
+ @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
+ public String principal="";
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (output != null && source != null) {
+ throw new BadCommandArgumentsException(DUPLICATE_ARGS);
+ }
+
+ // this is actually a !xor
+ if (keytab != null ^ !principal.isEmpty()) {
+ throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
new file mode 100644
index 0000000..e310f45
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
+
+public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_UPDATE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
new file mode 100644
index 0000000..00151f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Static argument manipulation operations
+ */
+public class ArgOps {
+
+ private static final Logger
+ log = LoggerFactory.getLogger(ArgOps.class);
+
+ /**
+ * create a 3-tuple
+ */
+ public static List<Object> triple(String msg, int min, int max) {
+ List<Object> l = new ArrayList<>(3);
+ l.add(msg);
+ l.add(min);
+ l.add(max);
+ return l;
+ }
+
+ public static void applyFileSystemBinding(String filesystemBinding,
+ Configuration conf) {
+ if (filesystemBinding != null) {
+ //filesystem argument was set -this overwrites any defaults in the
+ //configuration
+ FileSystem.setDefaultUri(conf, filesystemBinding);
+ }
+ }
+
+ public static void splitPairs(Collection<String> pairs,
+ Map<String, String> dest) {
+ for (String prop : pairs) {
+ String[] keyval = prop.split("=", 2);
+ if (keyval.length == 2) {
+ dest.put(keyval[0], keyval[1]);
+ }
+ }
+ }
+
+
+ public static void applyDefinitions(Map<String, String> definitionMap,
+ Configuration conf) {
+ for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ log.debug("configuration[{}]<=\"{}\"", key, val);
+ conf.set(key, val, "command line");
+ }
+ }
+
+ /**
+ * Create a map from a tuple list like ['worker','2','master','1] into a map
+ * ['worker':'2',"master":'1'];
+ * Duplicate entries also trigger errors
+ * @param description description for errors
+ * @param list list to conver to tuples
+ * @return the map of key value pairs -unordered.
+ * @throws BadCommandArgumentsException odd #of arguments received
+ */
+ public static Map<String, String> convertTupleListToMap(String description,
+ List<String> list) throws
+ BadCommandArgumentsException {
+ Map<String, String> results = new HashMap<>();
+ if (list != null && !list.isEmpty()) {
+ int size = list.size();
+ if (size % 2 != 0) {
+ //odd number of elements, not permitted
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_PARSE_FAILURE + description);
+ }
+ for (int count = 0; count < size; count += 2) {
+ String key = list.get(count);
+ String val = list.get(count + 1);
+ if (results.get(key) != null) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ + ": " + key);
+ }
+ results.put(key, val);
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Create a map from a tuple list like
+ * ['worker','heapsize','5G','master','heapsize','2M'] into a map
+ * ['worker':'2',"master":'1'];
+ * Duplicate entries also trigger errors
+
+ * @throws BadCommandArgumentsException odd #of arguments received
+ */
+ public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
+ List<String> list) throws BadCommandArgumentsException {
+
+ Map<String, Map<String, String>> results = new HashMap<>();
+ if (list != null && !list.isEmpty()) {
+ int size = list.size();
+ if (size % 3 != 0) {
+ //wrong number of elements, not permitted
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_PARSE_FAILURE + description);
+ }
+ for (int count = 0; count < size; count += 3) {
+ String role = list.get(count);
+ String key = list.get(count + 1);
+ String val = list.get(count + 2);
+ Map<String, String> roleMap = results.get(role);
+ if (roleMap == null) {
+ //demand create new role map
+ roleMap = new HashMap<>();
+ results.put(role, roleMap);
+ }
+ if (roleMap.get(key) != null) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ + ": for key " + key + " under " + role);
+ }
+ roleMap.put(key, val);
+ }
+ }
+ return results;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
new file mode 100644
index 0000000..204149b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+/**
+ * Here are all the arguments that may be parsed by the client or server
+ * command lines.
+ *
+ * Important: Please keep the main list in alphabetical order
+ * so it is easier to see what arguments are there
+ */
+public interface Arguments {
+
+ String ARG_APPDEF = "--appdef";
+ String ARG_BASE_PATH = "--basepath";
+ String ARG_COMPONENT = "--component";
+ String ARG_COMPONENT_SHORT = "--comp";
+ String ARG_COMPONENTS = "--components";
+ String ARG_COMP_OPT= "--compopt";
+ String ARG_COMP_OPT_SHORT = "--co";
+ String ARG_CONFIG = "--config";
+ String ARG_CONTAINERS = "--containers";
+ String ARG_DEBUG = "--debug";
+ String ARG_DEFINE = "-D";
+ String ARG_DELETE = "--delete";
+ String ARG_DEST = "--dest";
+ String ARG_DESTDIR = "--destdir";
+ String ARG_FILESYSTEM = "--fs";
+ String ARG_FILESYSTEM_LONG = "--filesystem";
+ String ARG_FOLDER = "--folder";
+ String ARG_FORCE = "--force";
+ String ARG_FORMAT = "--format";
+ String ARG_GETCONF = "--getconf";
+ String ARG_GETEXP = "--getexp";
+ String ARG_GETFILES = "--getfiles";
+ String ARG_HELP = "--help";
+ String ARG_IMAGE = "--image";
+ String ARG_INSTALL = "--install";
+ String ARG_INTERNAL = "--internal";
+ String ARG_KEYLEN = "--keylen";
+ String ARG_KEYTAB = "--keytab";
+ String ARG_KEYTABINSTALL = ARG_INSTALL;
+ String ARG_KEYTABDELETE = ARG_DELETE;
+ String ARG_KEYTABLIST = "--list";
+ String ARG_LIST = "--list";
+ String ARG_LISTCONF = "--listconf";
+ String ARG_LISTEXP = "--listexp";
+ String ARG_LISTFILES = "--listfiles";
+ String ARG_LIVE = "--live";
+ String ARG_MANAGER = "--manager";
+ String ARG_MANAGER_SHORT = "--m";
+ String ARG_MESSAGE = "--message";
+ String ARG_NAME = "--name";
+ String ARG_OPTION = "--option";
+ String ARG_OPTION_SHORT = "-O";
+ String ARG_OUTPUT = "--out";
+ String ARG_OUTPUT_SHORT = "-o";
+ String ARG_OVERWRITE = "--overwrite";
+ String ARG_PACKAGE = "--package";
+ String ARG_PATH = "--path";
+ String ARG_PRINCIPAL = "--principal";
+ String ARG_QUEUE = "--queue";
+ String ARG_LIFETIME = "--lifetime";
+ String ARG_RESOURCE = "--resource";
+ String ARG_RESOURCE_MANAGER = "--rm";
+ String ARG_SECURE = "--secure";
+ String ARG_SERVICETYPE = "--servicetype";
+ String ARG_SERVICES = "--services";
+ String ARG_SOURCE = "--source";
+ String ARG_STATE = "--state";
+ String ARG_SYSPROP = "-S";
+ String ARG_USER = "--user";
+ String ARG_UPLOAD = "--upload";
+ String ARG_VERBOSE = "--verbose";
+ String ARG_VERSION = "--version";
+ String ARG_WAIT = "--wait";
+/*
+ STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
+ RIGHT PLACE IN THE LIST
+ */
+
+ /**
+ * server: URI for the cluster
+ */
+ String ARG_CLUSTER_URI = "-cluster-uri";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
new file mode 100644
index 0000000..7b957fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+
+import java.util.Collection;
+
+/**
+ * Client CLI Args
+ */
+
+public class ClientArgs extends CommonArgs {
+
+ // =========================================================
+ // Keep all of these in alphabetical order. Thanks.
+ // =========================================================
+
+ private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
+ private final ActionClientArgs actionClientArgs = new ActionClientArgs();
+ private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
+ private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
+ private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
+ private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
+ private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
+ private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
+ private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
+ private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
+ private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
+ private final ActionListArgs actionListArgs = new ActionListArgs();
+ private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
+ private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
+ private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
+ private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
+ private final ActionThawArgs actionThawArgs = new ActionThawArgs();
+ private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
+ private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
+
+ public ClientArgs(String[] args) {
+ super(args);
+ }
+
+ public ClientArgs(Collection args) {
+ super(args);
+ }
+
+ @Override
+ protected void addActionArguments() {
+
+ addActions(
+ actionBuildArgs,
+ actionCreateArgs,
+ actionDependencyArgs,
+ actionDestroyArgs,
+ actionFlexArgs,
+ actionFreezeArgs,
+ actionHelpArgs,
+ actionStatusArgs,
+ actionThawArgs
+ );
+ }
+
+ @Override
+ public void applyDefinitions(Configuration conf) throws
+ BadCommandArgumentsException {
+ super.applyDefinitions(conf);
+ //RM
+ if (getManager() != null) {
+ log.debug("Setting RM to {}", getManager());
+ conf.set(YarnConfiguration.RM_ADDRESS, getManager());
+ }
+ if (getBasePath() != null) {
+ log.debug("Setting basePath to {}", getBasePath());
+ conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH,
+ getBasePath().toString());
+ }
+ }
+
+
+ public ActionBuildArgs getActionBuildArgs() {
+ return actionBuildArgs;
+ }
+
+ public ActionUpdateArgs getActionUpdateArgs() {
+ return actionUpdateArgs;
+ }
+
+ public ActionCreateArgs getActionCreateArgs() {
+ return actionCreateArgs;
+ }
+
+ public ActionDependencyArgs getActionDependencyArgs() {
+ return actionDependencyArgs;
+ }
+
+ public ActionDestroyArgs getActionDestroyArgs() {
+ return actionDestroyArgs;
+ }
+
+ public ActionExistsArgs getActionExistsArgs() {
+ return actionExistsArgs;
+ }
+
+ public ActionFlexArgs getActionFlexArgs() {
+ return actionFlexArgs;
+ }
+
+ public ActionFreezeArgs getActionFreezeArgs() {
+ return actionFreezeArgs;
+ }
+
+ public ActionListArgs getActionListArgs() {
+ return actionListArgs;
+ }
+
+
+ public ActionRegistryArgs getActionRegistryArgs() {
+ return actionRegistryArgs;
+ }
+
+ public ActionResolveArgs getActionResolveArgs() {
+ return actionResolveArgs;
+ }
+
+ public ActionResourceArgs getActionResourceArgs() {
+ return actionResourceArgs;
+ }
+
+ public ActionStatusArgs getActionStatusArgs() {
+ return actionStatusArgs;
+ }
+
+ public ActionThawArgs getActionThawArgs() {
+ return actionThawArgs;
+ }
+
+ public ActionTokensArgs getActionTokenArgs() {
+ return actionTokenArgs;
+ }
+
+ /**
+ * Look at the chosen action and bind it as the core action for the operation.
+ * @throws SliderException bad argument or similar
+ */
+ @Override
+ public void applyAction() throws SliderException {
+ String action = getAction();
+ if (SliderUtils.isUnset(action)) {
+ action = ACTION_HELP;
+ }
+ switch (action) {
+ case ACTION_BUILD:
+ bindCoreAction(actionBuildArgs);
+ break;
+
+ case ACTION_CREATE:
+ bindCoreAction(actionCreateArgs);
+ break;
+
+ case ACTION_STOP:
+ bindCoreAction(actionFreezeArgs);
+ break;
+
+ case ACTION_START:
+ bindCoreAction(actionThawArgs);
+ break;
+
+ case ACTION_DEPENDENCY:
+ bindCoreAction(actionDependencyArgs);
+ break;
+
+ case ACTION_DESTROY:
+ bindCoreAction(actionDestroyArgs);
+ break;
+
+ case ACTION_EXISTS:
+ bindCoreAction(actionExistsArgs);
+ break;
+
+ case ACTION_FLEX:
+ bindCoreAction(actionFlexArgs);
+ break;
+
+ case ACTION_HELP:
+ bindCoreAction(actionHelpArgs);
+ break;
+
+ case ACTION_KDIAG:
+ bindCoreAction(actionKDiagArgs);
+ break;
+
+ case ACTION_KEYTAB:
+ bindCoreAction(actionKeytabArgs);
+ break;
+
+ case ACTION_LIST:
+ bindCoreAction(actionListArgs);
+ break;
+
+ case ACTION_REGISTRY:
+ bindCoreAction(actionRegistryArgs);
+ break;
+
+ case ACTION_RESOLVE:
+ bindCoreAction(actionResolveArgs);
+ break;
+
+ case ACTION_RESOURCE:
+ bindCoreAction(actionResourceArgs);
+ break;
+
+ case ACTION_STATUS:
+ bindCoreAction(actionStatusArgs);
+ break;
+
+ case ACTION_TOKENS:
+ bindCoreAction(actionTokenArgs);
+ break;
+
+ case ACTION_UPDATE:
+ bindCoreAction(actionUpdateArgs);
+ break;
+
+ default:
+ throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ + " " + action);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
new file mode 100644
index 0000000..e1197ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterDescription;
+import com.beust.jcommander.ParameterException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class contains the common argument set for all tne entry points,
+ * and the core parsing logic to verify that the action is on the list
+ * of allowed actions -and that the remaining number of arguments is
+ * in the range allowed
+ */
+
+public abstract class CommonArgs extends ArgOps implements SliderActions,
+ Arguments {
+
+ protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
+
+
+ private static final int DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME = 30;
+
+
+ @Parameter(names = ARG_HELP, help = true)
+ public boolean help;
+
+
+ /**
+ -D name=value
+
+ Define an HBase configuration option which overrides any options in
+ the configuration XML files of the image or in the image configuration
+ directory. The values will be persisted.
+ Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
+
+ */
+
+ public Map<String, String> definitionMap = new HashMap<String, String>();
+ /**
+ * System properties
+ */
+ public Map<String, String> syspropsMap = new HashMap<String, String>();
+
+
+ /**
+ * fields
+ */
+ public final JCommander commander;
+ private final String[] args;
+
+ private AbstractActionArgs coreAction;
+
+ /**
+ * get the name: relies on arg 1 being the cluster name in all operations
+ * @return the name argument, null if there is none
+ */
+ public String getClusterName() {
+ return coreAction.getClusterName();
+ }
+
+ protected CommonArgs(String[] args) {
+ this.args = args;
+ commander = new JCommander(this);
+ }
+
+ protected CommonArgs(Collection args) {
+ List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
+ this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
+ commander = new JCommander(this);
+ }
+
+ public String usage() {
+ return usage(this, null);
+ }
+
+ public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
+ String result = null;
+ StringBuilder helperMessage = new StringBuilder();
+ if (commandOfInterest == null) {
+ // JCommander.usage is too verbose for a command with many options like
+ // slider no short version of that is found Instead, we compose our msg by
+ helperMessage.append("\nUsage: service COMMAND [options]\n");
+ helperMessage.append("where COMMAND is one of\n");
+ for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
+ helperMessage.append(String.format("\t%-"
+ + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", jcommand,
+ serviceArgs.commander.getCommandDescription(jcommand) + "\n"));
+ }
+ helperMessage
+ .append("Most commands print help when invoked without parameters or with --help");
+ result = helperMessage.toString();
+ } else {
+ helperMessage.append("\nUsage: service ").append(commandOfInterest);
+ helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <service>" : "");
+ helperMessage.append("\n");
+ for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
+ .get(commandOfInterest).getParameters()) {
+ String optional = paramDesc.getParameter().required() ? " (required)"
+ : " (optional)";
+ String paramName = paramDesc.getParameterized().getType() == Boolean.TYPE ? paramDesc
+ .getLongestName() : paramDesc.getLongestName() + " <"
+ + paramDesc.getParameterized().getName() + ">";
+ helperMessage.append(String.format("\t%-"
+ + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", paramName,
+ paramDesc.getDescription() + optional + "\n"));
+ result = helperMessage.toString();
+ }
+ }
+ return result;
+ }
+
+ public static String usage(CommonArgs serviceArgs) {
+ return usage(serviceArgs, null);
+ }
+
+ /**
+ * Parse routine -includes registering the action-specific argument classes
+ * and postprocess it
+ * @throws SliderException on any problem
+ */
+ public void parse() throws SliderException {
+ addActionArguments();
+ try {
+ commander.parse(args);
+ } catch (ParameterException e) {
+ throw new BadCommandArgumentsException(e, "%s in %s",
+ e.toString(),
+ (args != null
+ ? (SliderUtils.join(args,
+ " ", false))
+ : "[]"));
+ }
+ //now copy back to this class some of the attributes that are common to all
+ //actions
+ postProcess();
+ }
+
+ /**
+ * Add a command
+ * @param name action
+ * @param arg value
+ */
+ protected void addAction(String name, Object arg) {
+ commander.addCommand(name, arg);
+ }
+
+ protected void addActions(Object... actions) {
+ for (Object action : actions) {
+ commander.addCommand(action);
+ }
+ }
+
+ /**
+ * Override point to add a set of actions
+ */
+ protected void addActionArguments() {
+
+ }
+
+ /**
+ * validate args via {@link #validate()}
+ * then postprocess the arguments
+ */
+ public void postProcess() throws SliderException {
+ applyAction();
+ validate();
+
+ //apply entry set
+ for (Map.Entry<String, String> entry : syspropsMap.entrySet()) {
+ System.setProperty(entry.getKey(), entry.getValue());
+ }
+ }
+
+
+ /**
+ * Implementors must implement their action apply routine here
+ */
+ public abstract void applyAction() throws SliderException;
+
+
+ /**
+ * Bind the core action; this extracts any attributes that are used
+ * across routines
+ * @param action action to bind
+ */
+ protected void bindCoreAction(AbstractActionArgs action) {
+ coreAction = action;
+
+ splitPairs(coreAction.definitions, definitionMap);
+ splitPairs(coreAction.sysprops, syspropsMap);
+ }
+
+ /**
+ * Validate the arguments against the action requested
+ */
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ if (coreAction == null) {
+ throw new UsageException(ErrorStrings.ERROR_NO_ACTION + usage());
+ }
+ log.debug("action={}", getAction());
+ // let the action validate itself
+ try {
+ coreAction.validate();
+ } catch (BadCommandArgumentsException e) {
+ String badArgMsgBuilder =
+ e.getMessage() + System.lineSeparator() + usage(this,
+ coreAction.getActionName());
+ throw new BadCommandArgumentsException(badArgMsgBuilder);
+ }
+ }
+
+ /**
+ * Apply all the definitions on the command line to the configuration
+ * @param conf config
+ */
+ public void applyDefinitions(Configuration conf) throws
+ BadCommandArgumentsException {
+ applyDefinitions(definitionMap, conf);
+ }
+
+
+ /**
+ * If the Filesystem binding was provided, it overrides anything in
+ * the configuration
+ * @param conf configuration
+ */
+ public void applyFileSystemBinding(Configuration conf) {
+ ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf);
+ }
+
+ public boolean isDebug() {
+ return coreAction.debug;
+ }
+
+
+ public String getFilesystemBinding() {
+ return coreAction.filesystemBinding;
+ }
+
+ public Path getBasePath() { return coreAction.basePath; }
+
+ public String getManager() {
+ return coreAction.manager;
+ }
+
+ public String getAction() {
+ return commander.getParsedCommand();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
new file mode 100644
index 0000000..b6cd0a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ComponentArgsDelegate extends AbstractArgsDelegate {
+
+ /**
+ * This is a listing of the roles to create
+ */
+ @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT},
+ arity = 2,
+ description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
+ splitter = DontSplitArguments.class)
+ public List<String> componentTuples = new ArrayList<>(0);
+
+
+ /**
+ * Get the role mapping (may be empty, but never null)
+ * @return role mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
+ return convertTupleListToMap("component", componentTuples);
+ }
+
+ public List<String> getComponentTuples() {
+ return componentTuples;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
new file mode 100644
index 0000000..85de615
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.converters.IParameterSplitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DontSplitArguments implements IParameterSplitter {
+
+ @Override
+ public List<String> split(String value) {
+ List<String> list = new ArrayList<>(1);
+ list.add(value);
+ return list;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
new file mode 100644
index 0000000..bf194b6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import java.io.File;
+
+/**
+ * Launch args for create and start and anything else that can start something
+ */
+public interface LaunchArgsAccessor extends WaitTimeAccessor {
+ String getRmAddress();
+
+ File getOutputFile();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
new file mode 100644
index 0000000..d42510c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+
+import java.io.File;
+
+/**
+ * Any launch-time args
+ */
+public class LaunchArgsDelegate extends WaitArgsDelegate implements
+ LaunchArgsAccessor {
+
+
+ //TODO: do we need this?
+ @Parameter(names = ARG_RESOURCE_MANAGER,
+ description = "Resource manager hostname:port ",
+ required = false)
+ private String rmAddress;
+
+ @Override
+ public String getRmAddress() {
+ return rmAddress;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any service report")
+ public File outputFile;
+
+ @Override
+ public File getOutputFile() {
+ return outputFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
new file mode 100644
index 0000000..7972716
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Delegate for application and resource options.
+ */
+public class OptionArgsDelegate extends AbstractArgsDelegate {
+
+ /**
+ * Options key value.
+ */
+ @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
+ description = ARG_OPTION + "<name> <value>",
+ splitter = DontSplitArguments.class)
+ public List<String> optionTuples = new ArrayList<>(0);
+
+
+ /**
+ * All the app component option triples.
+ */
+ @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
+ description = "Component option " + ARG_COMP_OPT +
+ " <component> <name> <option>",
+ splitter = DontSplitArguments.class)
+ public List<String> compOptTriples = new ArrayList<>(0);
+
+ public Map<String, String> getOptionsMap() throws
+ BadCommandArgumentsException {
+ return convertTupleListToMap(ARG_OPTION, optionTuples);
+ }
+
+ /**
+ * Get the role heap mapping (may be empty, but never null).
+ * @return role heap mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, Map<String, String>> getCompOptionMap()
+ throws BadCommandArgumentsException {
+ return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
new file mode 100644
index 0000000..040ac64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+import org.apache.hadoop.fs.Path;
+
+public class PathArgumentConverter extends BaseConverter<Path> {
+
+ public PathArgumentConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public Path convert(String value) {
+ return new Path(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
new file mode 100644
index 0000000..1c38213
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.yarn.service.client.params;
+
+/**
+ * Parameters sent by the Client to the AM
+ */
+public class SliderAMArgs extends CommonArgs {
+
+ SliderAMCreateAction createAction = new SliderAMCreateAction();
+
+ public SliderAMArgs(String[] args) {
+ super(args);
+ }
+
+ @Override
+ protected void addActionArguments() {
+ addActions(createAction);
+ }
+
+ public String getImage() {
+ return createAction.image;
+ }
+
+ /**
+ * This is the URI in the FS to the Slider cluster; the conf file (and any
+ * other cluster-specifics) can be picked up here
+ */
+ public String getAppDefPath() {
+ return createAction.sliderClusterURI;
+ }
+
+ /**
+ * Am binding is simple: there is only one action
+ */
+ @Override
+ public void applyAction() {
+ bindCoreAction(createAction);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
new file mode 100644
index 0000000..a446665
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+import java.io.File;
+
+
+@Parameters(commandNames = { SliderActions.ACTION_CREATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class SliderAMCreateAction extends AbstractActionArgs implements
+ LaunchArgsAccessor {
+
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CREATE;
+ }
+
+ @Parameter(names = ARG_IMAGE, description = "image", required = false)
+ public String image;
+
+ /**
+ * This is the URI in the FS to the Slider cluster; the conf file (and any
+ * other cluster-specifics) can be picked up here
+ */
+ @Parameter(names = ARG_CLUSTER_URI,
+ description = "URI to the Slider cluster", required = true)
+ public String sliderClusterURI;
+
+ @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+ @Override
+ public String getRmAddress() {
+ return launchArgs.getRmAddress();
+ }
+
+ @Override
+ public int getWaittime() {
+ return launchArgs.getWaittime();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ launchArgs.setWaittime(waittime);
+ }
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
new file mode 100644
index 0000000..1b2a92d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+/**
+ * Actions.
+ * Only some of these are supported by specific Slider Services; they
+ * are listed here to ensure the names are consistent
+ */
+public interface SliderActions {
+ String ACTION_BUILD = "build";
+ String ACTION_CLIENT = "client";
+ String ACTION_CREATE = "create";
+ String ACTION_DEPENDENCY = "dependency";
+ String ACTION_UPDATE = "update";
+ String ACTION_UPGRADE = "upgrade";
+ String ACTION_DESTROY = "destroy";
+ String ACTION_EXISTS = "exists";
+ String ACTION_FLEX = "flex";
+ String ACTION_STOP = "stop";
+ String ACTION_HELP = "help";
+ String ACTION_INSTALL_KEYTAB = "install-keytab";
+ String ACTION_KDIAG = "kdiag";
+ String ACTION_KEYTAB = "keytab";
+ String ACTION_LIST = "list";
+
+ String ACTION_REGISTRY = "registry";
+ String ACTION_RESOLVE = "resolve";
+ String ACTION_RESOURCE = "resource";
+ String ACTION_STATUS = "status";
+ String ACTION_START = "start";
+ String ACTION_TOKENS = "tokens";
+
+ String DESCRIBE_ACTION_BUILD =
+ "Build a service specification, but do not start it";
+ String DESCRIBE_ACTION_CREATE =
+ "Build and start a service, it's equivalent to first invoke build and then start";
+ String DESCRIBE_ACTION_DEPENDENCY =
+ "Yarn service framework dependency (libraries) management";
+ String DESCRIBE_ACTION_UPDATE =
+ "Update template for service";
+ String DESCRIBE_ACTION_UPGRADE =
+ "Rolling upgrade/downgrade the component/containerto a newer/previous version";
+ String DESCRIBE_ACTION_DESTROY =
+ "Destroy a stopped service, service must be stopped first before destroying.";
+ String DESCRIBE_ACTION_EXISTS =
+ "Probe for a service running";
+ String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
+ String DESCRIBE_ACTION_FREEZE =
+ "Stop a running service";
+ String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
+ String DESCRIBE_ACTION_HELP = "Print help information";
+ String DESCRIBE_ACTION_LIST =
+ "List running services";
+ String DESCRIBE_ACTION_REGISTRY =
+ "Query the registry of a service";
+ String DESCRIBE_ACTION_STATUS =
+ "Get the status of a service";
+ String DESCRIBE_ACTION_THAW =
+ "Start a service with pre-built specification or a previously stopped service";
+ String DESCRIBE_ACTION_CLIENT = "Install the service client in the specified directory or obtain a client keystore or truststore";
+ String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
+ String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
+
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
new file mode 100644
index 0000000..86f3709
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+
+public class WaitArgsDelegate extends AbstractArgsDelegate implements
+ WaitTimeAccessor {
+
+
+ //--wait [timeout]
+ @Parameter(names = {ARG_WAIT},
+ description = "time to wait for an action to complete")
+ public int waittime = 0;
+
+
+ @Override
+ public int getWaittime() {
+ return waittime;
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ this.waittime = waittime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
new file mode 100644
index 0000000..f6afae6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+public interface WaitTimeAccessor {
+ int getWaittime();
+ void setWaittime(int waittime);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
deleted file mode 100644
index 8c968dc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ /dev/null
@@ -1,655 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.UpdatedContainer;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.component.ComponentEventType;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.hadoop.yarn.service.provider.ProviderUtils;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
-import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.text.MessageFormat;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
-import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
-import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
-import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
-
-/**
- *
- */
-public class ServiceScheduler extends CompositeService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceScheduler.class);
- private Application app;
-
- // component_name -> component
- private final Map<String, Component> componentsByName =
- new ConcurrentHashMap<>();
-
- // id - > component
- protected final Map<Long, Component> componentsById =
- new ConcurrentHashMap<>();
-
- private final Map<ContainerId, ComponentInstance> liveInstances =
- new ConcurrentHashMap<>();
-
- private ServiceMetrics serviceMetrics;
-
- private ServiceTimelinePublisher serviceTimelinePublisher;
-
- // Global diagnostics that will be reported to RM on eRxit.
- // The unit the number of characters. This will be limited to 64 * 1024
- // characters.
- private BoundedAppender diagnostics = new BoundedAppender(64 * 1024);
-
- // A cache for loading config files from remote such as hdfs
- public LoadingCache<ConfigFile, Object> configFileCache = null;
-
- public ScheduledExecutorService executorService;
- public Map<String, String> globalTokens = new HashMap<>();
-
- private AMRMClientAsync<AMRMClient.ContainerRequest> amRMClient;
- private NMClientAsync nmClient;
- private AsyncDispatcher dispatcher;
- AsyncDispatcher compInstanceDispatcher;
- private YarnRegistryViewForProviders yarnRegistryOperations;
- private ServiceContext context;
- private ContainerLaunchService containerLaunchService;
-
- public ServiceScheduler(ServiceContext context) {
- super(context.application.getName());
- this.context = context;
- }
-
- public void buildInstance(ServiceContext context, Configuration configuration)
- throws YarnException {
- app = context.application;
- executorService = Executors.newScheduledThreadPool(10);
- RegistryOperations registryClient = RegistryOperationsFactory
- .createInstance("ServiceScheduler", configuration);
- addIfService(registryClient);
- yarnRegistryOperations =
- createYarnRegistryOperations(context, registryClient);
-
- // register metrics
- serviceMetrics = ServiceMetrics
- .register(app.getName(), "Metrics for service");
- serviceMetrics.tag("type", "Metrics type [component or service]", "service");
- serviceMetrics.tag("appId", "Application id for service", app.getId());
-
- amRMClient = createAMRMClient();
- addIfService(amRMClient);
-
- nmClient = createNMClient();
- addIfService(nmClient);
-
- dispatcher = new AsyncDispatcher("Component dispatcher");
- dispatcher.register(ComponentEventType.class,
- new ComponentEventHandler());
- dispatcher.setDrainEventsOnStop();
- addIfService(dispatcher);
-
- compInstanceDispatcher =
- new AsyncDispatcher("CompInstance dispatcher");
- compInstanceDispatcher.register(ComponentInstanceEventType.class,
- new ComponentInstanceEventHandler());
- addIfService(compInstanceDispatcher);
- containerLaunchService = new ContainerLaunchService(context.fs);
- addService(containerLaunchService);
-
- if (YarnConfiguration.timelineServiceV2Enabled(configuration)) {
- TimelineV2Client timelineClient = TimelineV2Client
- .createTimelineClient(context.attemptId.getApplicationId());
- amRMClient.registerTimelineV2Client(timelineClient);
- serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
- addService(serviceTimelinePublisher);
- DefaultMetricsSystem.instance().register("ServiceMetricsSink",
- "For processing metrics to ATS",
- new ServiceMetricsSink(serviceTimelinePublisher));
- LOG.info("Timeline v2 is enabled.");
- }
-
- initGlobalTokensForSubstitute(context);
- //substitute quicklinks
- ProviderUtils.substituteMapWithTokens(app.getQuicklinks(), globalTokens);
- createConfigFileCache(context.fs.getFileSystem());
-
- createAllComponents();
- }
-
- protected YarnRegistryViewForProviders createYarnRegistryOperations(
- ServiceContext context, RegistryOperations registryClient) {
- return new YarnRegistryViewForProviders(registryClient,
- RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, app.getName(),
- context.attemptId);
- }
-
- protected NMClientAsync createNMClient() {
- return NMClientAsync.createNMClientAsync(new NMClientCallback());
- }
-
- protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
- return AMRMClientAsync
- .createAMRMClientAsync(1000, new AMRMClientCallback());
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- try {
- buildInstance(context, conf);
- } catch (YarnException e) {
- throw new YarnRuntimeException(e);
- }
- super.serviceInit(conf);
- }
-
- @Override
- public void serviceStop() throws Exception {
- LOG.info("Stopping service scheduler");
-
- if (executorService != null) {
- executorService.shutdownNow();
- }
-
- DefaultMetricsSystem.shutdown();
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- serviceTimelinePublisher
- .serviceAttemptUnregistered(context, diagnostics.toString());
- }
- // Cleanup each component instance. no need to release containers as
- // they will be automatically released by RM
- for (ComponentInstance instance : liveInstances.values()) {
- instance.cleanupRegistryAndCompHdfsDir();
- }
- String msg = diagnostics.toString()
- + "Navigate to the failed component for more details.";
- amRMClient
- .unregisterApplicationMaster(FinalApplicationStatus.ENDED, msg, "");
- LOG.info("Application " + app.getName()
- + " unregistered with RM, with attemptId = " + context.attemptId
- + ", diagnostics = " + diagnostics);
- super.serviceStop();
- }
-
- @Override
- public void serviceStart() throws Exception {
- super.serviceStart();
- InetSocketAddress bindAddress = context.clientAMService.getBindAddress();
- RegisterApplicationMasterResponse response = amRMClient
- .registerApplicationMaster(bindAddress.getHostName(),
- bindAddress.getPort(), "N/A");
- if (response.getClientToAMTokenMasterKey() != null
- && response.getClientToAMTokenMasterKey().remaining() != 0) {
- context.secretManager
- .setMasterKey(response.getClientToAMTokenMasterKey().array());
- }
- registerServiceInstance(context.attemptId, app);
-
- //TODO handle containers recover
- }
-
- private void recover() {
-
- }
-
- private void initGlobalTokensForSubstitute(ServiceContext context) {
- // ZK
- globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM, getConfig()
- .getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
- String user = null;
- try {
- user = UserGroupInformation.getCurrentUser().getShortUserName();
- } catch (IOException e) {
- LOG.error("Failed to get user.", e);
- }
- globalTokens
- .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkClusterPath(user, app.getName()));
-
- globalTokens.put(ServiceApiConstants.USER, user);
- String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);
- if (dnsDomain != null && !dnsDomain.isEmpty()) {
- globalTokens.put(ServiceApiConstants.DOMAIN, dnsDomain);
- }
- // HDFS
- String clusterFs = getConfig().getTrimmed(FS_DEFAULT_NAME_KEY);
- if (clusterFs != null && !clusterFs.isEmpty()) {
- globalTokens.put(ServiceApiConstants.CLUSTER_FS_URI, clusterFs);
- globalTokens.put(ServiceApiConstants.CLUSTER_FS_HOST,
- URI.create(clusterFs).getHost());
- }
- globalTokens.put(SERVICE_HDFS_DIR, context.serviceHdfsDir);
- // service name
- globalTokens.put(SERVICE_NAME_LC, app.getName().toLowerCase());
- globalTokens.put(SERVICE_NAME, app.getName());
- }
-
- private void createConfigFileCache(final FileSystem fileSystem) {
- this.configFileCache =
- CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES)
- .build(new CacheLoader<ConfigFile, Object>() {
- @Override public Object load(ConfigFile key) throws Exception {
- switch (key.getType()) {
- case HADOOP_XML:
- try (FSDataInputStream input = fileSystem
- .open(new Path(key.getSrcFile()))) {
- org.apache.hadoop.conf.Configuration confRead =
- new org.apache.hadoop.conf.Configuration(false);
- confRead.addResource(input);
- Map<String, String> map = new HashMap<>(confRead.size());
- for (Map.Entry<String, String> entry : confRead) {
- map.put(entry.getKey(), entry.getValue());
- }
- return map;
- }
- case TEMPLATE:
- try (FSDataInputStream fileInput = fileSystem
- .open(new Path(key.getSrcFile()))) {
- return IOUtils.toString(fileInput);
- }
- default:
- return null;
- }
- }
- });
- context.configCache = configFileCache;
- }
-
- private void registerServiceInstance(ApplicationAttemptId attemptId,
- Application application) throws IOException {
- LOG.info("Registering " + attemptId + ", " + application.getName()
- + " into registry");
- ServiceRecord serviceRecord = new ServiceRecord();
- serviceRecord.set(YarnRegistryAttributes.YARN_ID,
- attemptId.getApplicationId().toString());
- serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
- PersistencePolicies.APPLICATION);
- serviceRecord.description = "Yarn Service Master";
-
- serviceRecord.addExternalEndpoint(RegistryTypeUtils
- .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
- context.clientAMService.getBindAddress()));
-
- // set any provided attributes
- setUserProvidedServiceRecordAttributes(application.getConfiguration(),
- serviceRecord);
-
- executorService.submit(new Runnable() {
- @Override public void run() {
- try {
- yarnRegistryOperations.registerSelf(serviceRecord, true);
- LOG.info("Registered service under {}; absolute path {}",
- yarnRegistryOperations.getSelfRegistrationPath(),
- yarnRegistryOperations.getAbsoluteSelfRegistrationPath());
- boolean isFirstAttempt = 1 == attemptId.getAttemptId();
- // delete the children in case there are any and this is an AM startup.
- // just to make sure everything underneath is purged
- if (isFirstAttempt) {
- yarnRegistryOperations.deleteChildren(
- yarnRegistryOperations.getSelfRegistrationPath(), true);
- }
- } catch (IOException e) {
- LOG.error(
- "Failed to register app " + app.getName() + " in registry");
- }
- }
- });
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- serviceTimelinePublisher.serviceAttemptRegistered(app, getConfig());
- }
- }
-
- private void setUserProvidedServiceRecordAttributes(
- org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) {
- String prefix = "service.record.attribute";
- for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
- if (entry.getKey().startsWith(prefix)) {
- String key = entry.getKey().substring(prefix.length() + 1);
- record.set(key, entry.getValue().trim());
- }
- }
- }
-
- private void createAllComponents() {
- long allocateId = 0;
-
- // sort components by dependencies
- Collection<org.apache.hadoop.yarn.service.api.records.Component> sortedComponents =
- ServiceApiUtil.sortByDependencies(app.getComponents());
-
- for (org.apache.hadoop.yarn.service.api.records.Component compSpec : sortedComponents) {
- Component component = new Component(compSpec, allocateId, context);
- componentsById.put(allocateId, component);
- componentsByName.put(component.getName(), component);
- allocateId++;
-
- // Trigger the component without dependencies
- if (component.areDependenciesReady()) {
- ComponentEvent event = new ComponentEvent(compSpec.getName(), FLEX)
- .setDesired(compSpec.getNumberOfContainers());
- component.handle(event);
- }
- }
- }
-
- private final class ComponentEventHandler
- implements EventHandler<ComponentEvent> {
- @Override
- public void handle(ComponentEvent event) {
- Component component = componentsByName.get(event.getName());
-
- if (component == null) {
- LOG.error("No component exists for " + event.getName());
- return;
- }
- try {
- component.handle(event);
- } catch (Throwable t) {
- LOG.error(MessageFormat
- .format("[COMPONENT {0}]: Error in handling event type {1}",
- component.getName(), event.getType()), t);
- }
- }
- }
-
- private final class ComponentInstanceEventHandler
- implements EventHandler<ComponentInstanceEvent> {
- @Override
- public void handle(ComponentInstanceEvent event) {
- ComponentInstance instance =
- liveInstances.get(event.getContainerId());
- if (instance == null) {
- LOG.error("No component instance exists for " + event.getContainerId());
- return;
- }
- try {
- instance.handle(event);
- } catch (Throwable t) {
- LOG.error(instance.getCompInstanceId() +
- ": Error in handling event type " + event.getType(), t);
- }
- }
- }
-
- class AMRMClientCallback extends AMRMClientAsync.AbstractCallbackHandler {
-
- @Override
- public void onContainersAllocated(List<Container> containers) {
- LOG.info(containers.size() + " containers allocated. ");
- for (Container container : containers) {
- Component comp = componentsById.get(container.getAllocationRequestId());
- ComponentEvent event =
- new ComponentEvent(comp.getName(), CONTAINER_ALLOCATED)
- .setContainer(container);
- dispatcher.getEventHandler().handle(event);
- LOG.info("[COMPONENT {}]: {} outstanding container requests.",
- comp.getName(),
- amRMClient.getMatchingRequests(container.getAllocationRequestId()).size());
- // remove the corresponding request
- Collection<AMRMClient.ContainerRequest> collection = amRMClient
- .getMatchingRequests(container.getAllocationRequestId());
- if (collection.iterator().hasNext()) {
- AMRMClient.ContainerRequest request = collection.iterator().next();
- amRMClient.removeContainerRequest(request);
- }
-
- }
- }
-
- @Override
- public void onContainersCompleted(List<ContainerStatus> statuses) {
- for (ContainerStatus status : statuses) {
- ContainerId containerId = status.getContainerId();
- ComponentInstance instance = liveInstances.get(status.getContainerId());
- if (instance == null) {
- LOG.error(
- "Container {} Completed. No component instance exists. exitStatus={}. diagnostics={} ",
- containerId, status.getExitStatus(), status.getDiagnostics());
- return;
- }
- ComponentEvent event =
- new ComponentEvent(instance.getCompName(), CONTAINER_COMPLETED)
- .setStatus(status).setInstance(instance);
- dispatcher.getEventHandler().handle(event);
- }
- }
-
- @Override
- public void onContainersUpdated(List<UpdatedContainer> containers) {
- }
-
- @Override public void onShutdownRequest() {
- //Was used for non-work-preserving restart in YARN, should be deprecated.
- }
-
- @Override public void onNodesUpdated(List<NodeReport> updatedNodes) {
- StringBuilder str = new StringBuilder();
- str.append("Nodes updated info: ").append(System.lineSeparator());
- for (NodeReport report : updatedNodes) {
- str.append(report.getNodeId()).append(", state = ")
- .append(report.getNodeState()).append(", healthDiagnostics = ")
- .append(report.getHealthReport()).append(System.lineSeparator());
- }
- LOG.warn(str.toString());
- }
-
- @Override public float getProgress() {
- // get running containers over desired containers
- long total = 0;
- for (org.apache.hadoop.yarn.service.api.records.Component component : app
- .getComponents()) {
- total += component.getNumberOfContainers();
- }
- // Probably due to user flexed down to 0
- if (total == 0) {
- return 100;
- }
- return Math.max((float) liveInstances.size() / total * 100, 100);
- }
-
- @Override public void onError(Throwable e) {
- LOG.error("Error in AMRMClient callback handler ", e);
- }
- }
-
-
- private class NMClientCallback extends NMClientAsync.AbstractCallbackHandler {
-
- @Override public void onContainerStarted(ContainerId containerId,
- Map<String, ByteBuffer> allServiceResponse) {
- ComponentInstance instance = liveInstances.get(containerId);
- if (instance == null) {
- LOG.error("No component instance exists for " + containerId);
- return;
- }
- ComponentEvent event =
- new ComponentEvent(instance.getCompName(), CONTAINER_STARTED)
- .setInstance(instance);
- dispatcher.getEventHandler().handle(event);
- }
-
- @Override public void onContainerStatusReceived(ContainerId containerId,
- ContainerStatus containerStatus) {
-
- }
-
- @Override public void onContainerStopped(ContainerId containerId) {
-
- }
-
- @Override
- public void onStartContainerError(ContainerId containerId, Throwable t) {
- ComponentInstance instance = liveInstances.get(containerId);
- if (instance == null) {
- LOG.error("No component instance exists for " + containerId);
- return;
- }
- amRMClient.releaseAssignedContainer(containerId);
- // After container released, it'll get CONTAINER_COMPLETED event from RM
- // automatically which will trigger stopping COMPONENT INSTANCE
- }
-
- @Override public void onContainerResourceIncreased(ContainerId containerId,
- Resource resource) {
-
- }
-
- @Override public void onGetContainerStatusError(ContainerId containerId,
- Throwable t) {
-
- }
-
- @Override
- public void onIncreaseContainerResourceError(ContainerId containerId,
- Throwable t) {
-
- }
-
- @Override
- public void onStopContainerError(ContainerId containerId, Throwable t) {
-
- }
- }
-
- public ServiceMetrics getServiceMetrics() {
- return serviceMetrics;
- }
-
- public AMRMClientAsync<AMRMClient.ContainerRequest> getAmRMClient() {
- return amRMClient;
- }
-
- public NMClientAsync getNmClient() {
- return nmClient;
- }
-
- public void addLiveCompInstance(ContainerId containerId,
- ComponentInstance instance) {
- liveInstances.put(containerId, instance);
- }
-
- public void removeLiveCompInstance(ContainerId containerId) {
- liveInstances.remove(containerId);
- }
-
- public AsyncDispatcher getCompInstanceDispatcher() {
- return compInstanceDispatcher;
- }
-
- public YarnRegistryViewForProviders getYarnRegistryOperations() {
- return yarnRegistryOperations;
- }
-
- public ServiceTimelinePublisher getServiceTimelinePublisher() {
- return serviceTimelinePublisher;
- }
-
- public Map<ContainerId, ComponentInstance> getLiveInstances() {
- return liveInstances;
- }
-
- public ContainerLaunchService getContainerLaunchService() {
- return containerLaunchService;
- }
-
- public ServiceContext getContext() {
- return context;
- }
-
- public Map<String, Component> getAllComponents() {
- return componentsByName;
- }
-
- public Application getApp() {
- return app;
- }
-
- public AsyncDispatcher getDispatcher() {
- return dispatcher;
- }
-
- public BoundedAppender getDiagnostics() {
- return diagnostics;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
deleted file mode 100644
index cf9e31f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.constants;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
-
-/**
- * This class defines constants that can be used in input spec for
- * variable substitutions
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public interface ServiceApiConstants {
-
- // Constants for service
- String SERVICE_NAME = $("SERVICE_NAME");
-
- String SERVICE_NAME_LC = $("SERVICE_NAME.lc");
-
- String USER = $("USER");
-
- String DOMAIN = $("DOMAIN");
-
- // Constants for component
- String COMPONENT_NAME = $("COMPONENT_NAME");
-
- String COMPONENT_NAME_LC = $("COMPONENT_NAME.lc");
-
- String COMPONENT_INSTANCE_NAME = $("COMPONENT_INSTANCE_NAME");
-
- // Constants for component instance
- String COMPONENT_ID = $("COMPONENT_ID");
-
- String CONTAINER_ID = $("CONTAINER_ID");
-
- // Constants for default cluster ZK
- String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
-
- // URI for the default cluster fs
- String CLUSTER_FS_URI = $("CLUSTER_FS_URI");
-
- // the host component of the cluster fs UI
- String CLUSTER_FS_HOST = $("CLUSTER_FS_HOST");
-
- // Path in zookeeper for a specific service
- String SERVICE_ZK_PATH = $("SERVICE_ZK_PATH");
-
- // Constants for service specific hdfs dir
- String SERVICE_HDFS_DIR = $("SERVICE_HDFS_DIR");
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
deleted file mode 100644
index f9e5154..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * An Application resource has the following attributes.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "An Application resource has the following attributes.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
- "lifetime", "containers" })
-public class Application extends BaseResource {
- private static final long serialVersionUID = -4491694636566094885L;
-
- private String name = null;
- private String id = null;
- private Artifact artifact = null;
- private Resource resource = null;
- private String launchCommand = null;
- private Date launchTime = null;
- private Long numberOfContainers = null;
- private Long numberOfRunningContainers = null;
- private Long lifetime = null;
- private PlacementPolicy placementPolicy = null;
- private List<Component> components = new ArrayList<>();
- private Configuration configuration = new Configuration();
- private List<Container> containers = new ArrayList<>();
- private ApplicationState state = null;
- private Map<String, String> quicklinks = new HashMap<>();
- private String queue = null;
-
- /**
- * A unique application name.
- **/
- public Application name(String name) {
- this.name = name;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "A unique application name.")
- @JsonProperty("name")
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * A unique application id.
- **/
- public Application id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A unique application id.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * Artifact of single-component applications. Mandatory if components
- * attribute is not specified.
- **/
- public Application artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact of single-component applications. Mandatory if components attribute is not specified.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- /**
- * Resource of single-component applications or the global default for
- * multi-component applications. Mandatory if it is a single-component
- * application and if cpus and memory are not specified at the Application
- * level.
- **/
- public Application resource(Resource resource) {
- this.resource = resource;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.")
- @JsonProperty("resource")
- public Resource getResource() {
- return resource;
- }
-
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- /**
- * The custom launch command of an application component (optional). If not
- * specified for applications with docker images say, it will default to the
- * default start command of the image. If there is a single component in this
- * application, you can specify this without the need to have a 'components'
- * section.
- **/
- public Application launchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.")
- @JsonProperty("launch_command")
- public String getLaunchCommand() {
- return launchCommand;
- }
-
- @XmlElement(name = "launch_command")
- public void setLaunchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- }
-
- /**
- * The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
- **/
- public Application launchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
- @JsonProperty("launch_time")
- public Date getLaunchTime() {
- return launchTime == null ? null : (Date) launchTime.clone();
- }
-
- @XmlElement(name = "launch_time")
- public void setLaunchTime(Date launchTime) {
- this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
- }
-
- /**
- * Number of containers for each app-component in the application. Each
- * app-component can further override this app-level global default.
- **/
- public Application numberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Number of containers for each app-component in the application. Each app-component can further override this app-level global default.")
- @JsonProperty("number_of_containers")
- public Long getNumberOfContainers() {
- return numberOfContainers;
- }
-
- @XmlElement(name = "number_of_containers")
- public void setNumberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- }
-
- /**
- * In get response this provides the total number of running containers for
- * this application (across all components) at the time of request. Note, a
- * subsequent request can return a different number as and when more
- * containers get allocated until it reaches the total number of containers or
- * if a flex request has been made between the two requests.
- **/
- public Application numberOfRunningContainers(Long numberOfRunningContainers) {
- this.numberOfRunningContainers = numberOfRunningContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
- @JsonProperty("number_of_running_containers")
- public Long getNumberOfRunningContainers() {
- return numberOfRunningContainers;
- }
-
- @XmlElement(name = "number_of_running_containers")
- public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
- this.numberOfRunningContainers = numberOfRunningContainers;
- }
-
- /**
- * Life time (in seconds) of the application from the time it reaches the
- * RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For
- * unlimited lifetime do not set a lifetime value.
- **/
- public Application lifetime(Long lifetime) {
- this.lifetime = lifetime;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Life time (in seconds) of the application from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.")
- @JsonProperty("lifetime")
- public Long getLifetime() {
- return lifetime;
- }
-
- public void setLifetime(Long lifetime) {
- this.lifetime = lifetime;
- }
-
- /**
- * Advanced scheduling and placement policies (optional). If not specified, it
- * defaults to the default placement policy of the app owner. The design of
- * placement policies are in the works. It is not very clear at this point,
- * how policies in conjunction with labels be exposed to application owners.
- * This is a placeholder for now. The advanced structure of this attribute
- * will be determined by YARN-4902.
- **/
- public Application placementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
- @JsonProperty("placement_policy")
- public PlacementPolicy getPlacementPolicy() {
- return placementPolicy;
- }
-
- @XmlElement(name = "placement_policy")
- public void setPlacementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- }
-
- /**
- * Components of an application.
- **/
- public Application components(List<Component> components) {
- this.components = components;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Components of an application.")
- @JsonProperty("components")
- public List<Component> getComponents() {
- return components;
- }
-
- public void setComponents(List<Component> components) {
- this.components = components;
- }
-
- public void addComponent(Component component) {
- components.add(component);
- }
-
- public Component getComponent(String name) {
- for (Component component : components) {
- if (component.getName().equals(name)) {
- return component;
- }
- }
- return null;
- }
-
- /**
- * Config properties of an application. Configurations provided at the
- * application/global level are available to all the components. Specific
- * properties can be overridden at the component level.
- **/
- public Application configuration(Configuration configuration) {
- this.configuration = configuration;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.")
- @JsonProperty("configuration")
- public Configuration getConfiguration() {
- return configuration;
- }
-
- public void setConfiguration(Configuration configuration) {
- this.configuration = configuration;
- }
-
- /**
- * Containers of a started application. Specifying a value for this attribute
- * for the POST payload raises a validation error. This blob is available only
- * in the GET response of a started application.
- **/
- public Application containers(List<Container> containers) {
- this.containers = containers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
- @JsonProperty("containers")
- public List<Container> getContainers() {
- return containers;
- }
-
- public void setContainers(List<Container> containers) {
- this.containers = containers;
- }
-
- public void addContainer(Container container) {
- this.containers.add(container);
- }
-
- /**
- * State of the application. Specifying a value for this attribute for the
- * POST payload raises a validation error. This attribute is available only in
- * the GET response of a started application.
- **/
- public Application state(ApplicationState state) {
- this.state = state;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.")
- @JsonProperty("state")
- public ApplicationState getState() {
- return state;
- }
-
- public void setState(ApplicationState state) {
- this.state = state;
- }
-
- /**
- * A blob of key-value pairs of quicklinks to be exported for an application.
- **/
- public Application quicklinks(Map<String, String> quicklinks) {
- this.quicklinks = quicklinks;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an application.")
- @JsonProperty("quicklinks")
- public Map<String, String> getQuicklinks() {
- return quicklinks;
- }
-
- public void setQuicklinks(Map<String, String> quicklinks) {
- this.quicklinks = quicklinks;
- }
-
- /**
- * The YARN queue that this application should be submitted to.
- **/
- public Application queue(String queue) {
- this.queue = queue;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The YARN queue that this application should be submitted to.")
- @JsonProperty("queue")
- public String getQueue() {
- return queue;
- }
-
- public void setQueue(String queue) {
- this.queue = queue;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Application application = (Application) o;
- return Objects.equals(this.name, application.name);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(name);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Application {\n");
-
- sb.append(" name: ").append(toIndentedString(name)).append("\n");
- sb.append(" id: ").append(toIndentedString(id)).append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
- .append("\n");
- sb.append(" launchTime: ").append(toIndentedString(launchTime))
- .append("\n");
- sb.append(" numberOfContainers: ")
- .append(toIndentedString(numberOfContainers)).append("\n");
- sb.append(" numberOfRunningContainers: ")
- .append(toIndentedString(numberOfRunningContainers)).append("\n");
- sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n");
- sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
- .append("\n");
- sb.append(" components: ").append(toIndentedString(components))
- .append("\n");
- sb.append(" configuration: ").append(toIndentedString(configuration))
- .append("\n");
- sb.append(" containers: ").append(toIndentedString(containers))
- .append("\n");
- sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
- .append("\n");
- sb.append(" queue: ").append(toIndentedString(queue)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
deleted file mode 100644
index acef562..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * The current state of an application.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "The current state of an application.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public enum ApplicationState {
- ACCEPTED, STARTED, READY, STOPPED, FAILED;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
deleted file mode 100644
index b57225a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.util.Objects;
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * The current status of a submitted application, returned as a response to the
- * GET API.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "The current status of a submitted application, returned as a response to the GET API.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ApplicationStatus extends BaseResource {
- private static final long serialVersionUID = -3469885905347851034L;
-
- private String diagnostics = null;
- private ApplicationState state = null;
- private Integer code = null;
-
- /**
- * Diagnostic information (if any) for the reason of the current state of the
- * application. It typically has a non-null value, if the application is in a
- * non-running state.
- **/
- public ApplicationStatus diagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.")
- @JsonProperty("diagnostics")
- public String getDiagnostics() {
- return diagnostics;
- }
-
- public void setDiagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
- }
-
- /**
- * Application state.
- **/
- public ApplicationStatus state(ApplicationState state) {
- this.state = state;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Application state.")
- @JsonProperty("state")
- public ApplicationState getState() {
- return state;
- }
-
- public void setState(ApplicationState state) {
- this.state = state;
- }
-
- /**
- * An error code specific to a scenario which app owners should be able to use
- * to understand the failure in addition to the diagnostic information.
- **/
- public ApplicationStatus code(Integer code) {
- this.code = code;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.")
- @JsonProperty("code")
- public Integer getCode() {
- return code;
- }
-
- public void setCode(Integer code) {
- this.code = code;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ApplicationStatus applicationStatus = (ApplicationStatus) o;
- return Objects.equals(this.diagnostics, applicationStatus.diagnostics)
- && Objects.equals(this.state, applicationStatus.state)
- && Objects.equals(this.code, applicationStatus.code);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(diagnostics, state, code);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class ApplicationStatus {\n");
-
- sb.append(" diagnostics: ").append(toIndentedString(diagnostics))
- .append("\n");
- sb.append(" state: ").append(toIndentedString(state)).append("\n");
- sb.append(" code: ").append(toIndentedString(code)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
deleted file mode 100644
index 0ddc374..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Artifact of an application component.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "Artifact of an application component")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Artifact implements Serializable {
- private static final long serialVersionUID = 3608929500111099035L;
-
- private String id = null;
-
- public enum TypeEnum {
- DOCKER("DOCKER"), TARBALL("TARBALL"), APPLICATION("APPLICATION");
-
- private String value;
-
- TypeEnum(String value) {
- this.value = value;
- }
-
- @Override
- @JsonValue
- public String toString() {
- return value;
- }
- }
-
- private TypeEnum type = TypeEnum.DOCKER;
- private String uri = null;
-
- /**
- * Artifact id. Examples are package location uri for tarball based apps,
- * image name for docker, etc.
- **/
- public Artifact id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "Artifact id. Examples are package location uri for tarball based apps, image name for docker, etc.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * Artifact type, like docker, tarball, etc. (optional).
- **/
- public Artifact type(TypeEnum type) {
- this.type = type;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact type, like docker, tarball, etc. (optional).")
- @JsonProperty("type")
- public TypeEnum getType() {
- return type;
- }
-
- public void setType(TypeEnum type) {
- this.type = type;
- }
-
- /**
- * Artifact location to support multiple artifact stores (optional).
- **/
- public Artifact uri(String uri) {
- this.uri = uri;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact location to support multiple artifact stores (optional).")
- @JsonProperty("uri")
- public String getUri() {
- return uri;
- }
-
- public void setUri(String uri) {
- this.uri = uri;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Artifact artifact = (Artifact) o;
- return Objects.equals(this.id, artifact.id)
- && Objects.equals(this.type, artifact.type)
- && Objects.equals(this.uri, artifact.uri);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(id, type, uri);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Artifact {\n");
-
- sb.append(" id: ").append(toIndentedString(id)).append("\n");
- sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" uri: ").append(toIndentedString(uri)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
deleted file mode 100644
index a87c97f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import java.io.Serializable;
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class BaseResource implements Serializable {
- private static final long serialVersionUID = 1492603053176889431L;
-
- private String uri;
-
- /**
- * Resource location, e.g. \
- * "/applications/helloworld/containers/container_e3751_1458061340047_0008_01_000002\
- * "
- **/
- public String getUri() {
- return uri;
- }
-
- public void setUri(String uri) {
- this.uri = uri;
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("BaseResource [uri=");
- builder.append(uri);
- builder.append("]");
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
deleted file mode 100644
index 633e862..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * One or more components of the application. If the application is HBase say,
- * then the component can be a simple role like master or regionserver. If the
- * application is a complex business webapp then a component can be other
- * applications say Kafka or Storm. Thereby it opens up the support for complex
- * and nested applications.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Component implements Serializable {
- private static final long serialVersionUID = -8430058381509087805L;
-
- private String name = null;
- private List<String> dependencies = new ArrayList<String>();
- private ReadinessCheck readinessCheck = null;
- private Artifact artifact = null;
- private String launchCommand = null;
- private Resource resource = null;
- private Long numberOfContainers = null;
- private Boolean runPrivilegedContainer = false;
- private PlacementPolicy placementPolicy = null;
- private Configuration configuration = new Configuration();
- private List<String> quicklinks = new ArrayList<String>();
- private List<Container> containers =
- Collections.synchronizedList(new ArrayList<Container>());
-
- /**
- * Name of the application component (mandatory).
- **/
- public Component name(String name) {
- this.name = name;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "Name of the application component (mandatory).")
- @JsonProperty("name")
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * An array of application components which should be in READY state (as
- * defined by readiness check), before this component can be started. The
- * dependencies across all components of an application should be represented
- * as a DAG.
- **/
- public Component dependencies(List<String> dependencies) {
- this.dependencies = dependencies;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.")
- @JsonProperty("dependencies")
- public List<String> getDependencies() {
- return dependencies;
- }
-
- public void setDependencies(List<String> dependencies) {
- this.dependencies = dependencies;
- }
-
- /**
- * Readiness check for this app-component.
- **/
- public Component readinessCheck(ReadinessCheck readinessCheck) {
- this.readinessCheck = readinessCheck;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Readiness check for this app-component.")
- @JsonProperty("readiness_check")
- public ReadinessCheck getReadinessCheck() {
- return readinessCheck;
- }
-
- @XmlElement(name = "readiness_check")
- public void setReadinessCheck(ReadinessCheck readinessCheck) {
- this.readinessCheck = readinessCheck;
- }
-
- /**
- * Artifact of the component (optional). If not specified, the application
- * level global artifact takes effect.
- **/
- public Component artifact(Artifact artifact) {
- this.artifact = artifact;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the application level global artifact takes effect.")
- @JsonProperty("artifact")
- public Artifact getArtifact() {
- return artifact;
- }
-
- public void setArtifact(Artifact artifact) {
- this.artifact = artifact;
- }
-
- /**
- * The custom launch command of this component (optional). When specified at
- * the component level, it overrides the value specified at the global level
- * (if any).
- **/
- public Component launchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
- @JsonProperty("launch_command")
- public String getLaunchCommand() {
- return launchCommand;
- }
-
- @XmlElement(name = "launch_command")
- public void setLaunchCommand(String launchCommand) {
- this.launchCommand = launchCommand;
- }
-
- /**
- * Resource of this component (optional). If not specified, the application
- * level global resource takes effect.
- **/
- public Component resource(Resource resource) {
- this.resource = resource;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the application level global resource takes effect.")
- @JsonProperty("resource")
- public Resource getResource() {
- return resource;
- }
-
- public void setResource(Resource resource) {
- this.resource = resource;
- }
-
- /**
- * Number of containers for this app-component (optional). If not specified,
- * the application level global number_of_containers takes effect.
- **/
- public Component numberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.")
- @JsonProperty("number_of_containers")
- public Long getNumberOfContainers() {
- return numberOfContainers;
- }
-
- @XmlElement(name = "number_of_containers")
- public void setNumberOfContainers(Long numberOfContainers) {
- this.numberOfContainers = numberOfContainers;
- }
-
- @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
- @JsonProperty("containers")
- public List<Container> getContainers() {
- return containers;
- }
-
- public void setContainers(List<Container> containers) {
- this.containers = containers;
- }
-
- public void addContainer(Container container) {
- this.containers.add(container);
- }
-
- public void removeContainer(Container container) {
- containers.remove(container);
- }
- public Container getContainer(String id) {
- for (Container container : containers) {
- if (container.getId().equals(id)) {
- return container;
- }
- }
- return null;
- }
-
- /**
- * Run all containers of this component in privileged mode (YARN-4262).
- **/
- public Component runPrivilegedContainer(Boolean runPrivilegedContainer) {
- this.runPrivilegedContainer = runPrivilegedContainer;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
- @JsonProperty("run_privileged_container")
- public Boolean getRunPrivilegedContainer() {
- return runPrivilegedContainer;
- }
-
- @XmlElement(name = "run_privileged_container")
- public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
- this.runPrivilegedContainer = runPrivilegedContainer;
- }
-
- /**
- * Advanced scheduling and placement policies for all containers of this
- * component (optional). If not specified, the app level placement_policy
- * takes effect. Refer to the description at the global level for more
- * details.
- **/
- public Component placementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.")
- @JsonProperty("placement_policy")
- public PlacementPolicy getPlacementPolicy() {
- return placementPolicy;
- }
-
- @XmlElement(name = "placement_policy")
- public void setPlacementPolicy(PlacementPolicy placementPolicy) {
- this.placementPolicy = placementPolicy;
- }
-
- /**
- * Config properties for this app-component.
- **/
- public Component configuration(Configuration configuration) {
- this.configuration = configuration;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Config properties for this app-component.")
- @JsonProperty("configuration")
- public Configuration getConfiguration() {
- return configuration;
- }
-
- public void setConfiguration(Configuration configuration) {
- this.configuration = configuration;
- }
-
- /**
- * A list of quicklink keys defined at the application level, and to be
- * resolved by this component.
- **/
- public Component quicklinks(List<String> quicklinks) {
- this.quicklinks = quicklinks;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the application level, and to be resolved by this component.")
- @JsonProperty("quicklinks")
- public List<String> getQuicklinks() {
- return quicklinks;
- }
-
- public void setQuicklinks(List<String> quicklinks) {
- this.quicklinks = quicklinks;
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Component component = (Component) o;
- return Objects.equals(this.name, component.name)
- && Objects.equals(this.dependencies, component.dependencies)
- && Objects.equals(this.readinessCheck, component.readinessCheck)
- && Objects.equals(this.artifact, component.artifact)
- && Objects.equals(this.launchCommand, component.launchCommand)
- && Objects.equals(this.resource, component.resource)
- && Objects.equals(this.numberOfContainers, component.numberOfContainers)
- && Objects.equals(this.runPrivilegedContainer,
- component.runPrivilegedContainer)
- && Objects.equals(this.placementPolicy, component.placementPolicy)
- && Objects.equals(this.configuration, component.configuration)
- && Objects.equals(this.quicklinks, component.quicklinks);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(name, dependencies, readinessCheck, artifact,
- launchCommand, resource, numberOfContainers,
- runPrivilegedContainer, placementPolicy, configuration, quicklinks);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class Component {\n");
-
- sb.append(" name: ").append(toIndentedString(name)).append("\n");
- sb.append(" dependencies: ").append(toIndentedString(dependencies))
- .append("\n");
- sb.append(" readinessCheck: ").append(toIndentedString(readinessCheck))
- .append("\n");
- sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
- sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
- .append("\n");
- sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
- sb.append(" numberOfContainers: ")
- .append(toIndentedString(numberOfContainers)).append("\n");
- sb.append(" containers: ").append(toIndentedString(containers))
- .append("\n");
- sb.append(" runPrivilegedContainer: ")
- .append(toIndentedString(runPrivilegedContainer)).append("\n");
- sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
- .append("\n");
- sb.append(" configuration: ").append(toIndentedString(configuration))
- .append("\n");
- sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
- .append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-
- /**
- * Merge from another component into this component without overwriting.
- */
- public void mergeFrom(Component that) {
- if (this.getArtifact() == null) {
- this.setArtifact(that.getArtifact());
- }
- if (this.getResource() == null) {
- this.setResource(that.getResource());
- }
- if (this.getNumberOfContainers() == null) {
- this.setNumberOfContainers(that.getNumberOfContainers());
- }
- if (this.getLaunchCommand() == null) {
- this.setLaunchCommand(that.getLaunchCommand());
- }
- this.getConfiguration().mergeFrom(that.getConfiguration());
- if (this.getQuicklinks() == null) {
- this.setQuicklinks(that.getQuicklinks());
- }
- if (this.getRunPrivilegedContainer() == null) {
- this.setRunPrivilegedContainer(that.getRunPrivilegedContainer());
- }
- if (this.getDependencies() == null) {
- this.setDependencies(that.getDependencies());
- }
- if (this.getPlacementPolicy() == null) {
- this.setPlacementPolicy(that.getPlacementPolicy());
- }
- if (this.getReadinessCheck() == null) {
- this.setReadinessCheck(that.getReadinessCheck());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
deleted file mode 100644
index 2fb494e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonValue;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * A config file that needs to be created and made available as a volume in an
- * application component container.
- **/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@ApiModel(description = "A config file that needs to be created and made available as a volume in an application component container.")
-@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-@XmlRootElement
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ConfigFile implements Serializable {
- private static final long serialVersionUID = -7009402089417704612L;
-
- public enum TypeEnum {
- XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
- "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
-
- private String value;
-
- TypeEnum(String value) {
- this.value = value;
- }
-
- @Override
- @JsonValue
- public String toString() {
- return value;
- }
- }
-
- private TypeEnum type = null;
- private String destFile = null;
- private String srcFile = null;
- private Map<String, String> props = new HashMap<>();
-
- public ConfigFile copy() {
- ConfigFile copy = new ConfigFile();
- copy.setType(this.getType());
- copy.setSrcFile(this.getSrcFile());
- copy.setDestFile(this.getDestFile());
- if (this.getProps() != null && !this.getProps().isEmpty()) {
- copy.getProps().putAll(this.getProps());
- }
- return copy;
- }
-
- /**
- * Config file in the standard format like xml, properties, json, yaml,
- * template.
- **/
- public ConfigFile type(TypeEnum type) {
- this.type = type;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "Config file in the standard format like xml, properties, json, yaml, template.")
- @JsonProperty("type")
- public TypeEnum getType() {
- return type;
- }
-
- public void setType(TypeEnum type) {
- this.type = type;
- }
-
- /**
- * The absolute path that this configuration file should be mounted as, in the
- * application container.
- **/
- public ConfigFile destFile(String destFile) {
- this.destFile = destFile;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "The absolute path that this configuration file should be mounted as, in the application container.")
- @JsonProperty("dest_file")
- public String getDestFile() {
- return destFile;
- }
-
- @XmlElement(name = "dest_file")
- public void setDestFile(String destFile) {
- this.destFile = destFile;
- }
-
- /**
- * This provides the source location of the configuration file, the content
- * of which is dumped to dest_file post property substitutions, in the format
- * as specified in type. Typically the src_file would point to a source
- * controlled network accessible file maintained by tools like puppet, chef,
- * or hdfs etc. Currently, only hdfs is supported.
- **/
- public ConfigFile srcFile(String srcFile) {
- this.srcFile = srcFile;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "This provides the source location of the configuration file, "
- + "the content of which is dumped to dest_file post property substitutions, in the format as specified in type. "
- + "Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.")
- @JsonProperty("src_file")
- public String getSrcFile() {
- return srcFile;
- }
-
- @XmlElement(name = "src_file")
- public void setSrcFile(String srcFile) {
- this.srcFile = srcFile;
- }
-
- /**
- A blob of key value pairs that will be dumped in the dest_file in the format
- as specified in type. If src_file is specified, src_file content are dumped
- in the dest_file and these properties will overwrite, if any, existing
- properties in src_file or be added as new properties in src_file.
- **/
- public ConfigFile props(Map<String, String> props) {
- this.props = props;
- return this;
- }
-
- @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type."
- + " If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any,"
- + " existing properties in src_file or be added as new properties in src_file.")
- @JsonProperty("props")
- public Map<String, String> getProps() {
- return props;
- }
-
- public void setProps(Map<String, String> props) {
- this.props = props;
- }
-
- public long getLong(String name, long defaultValue) {
- if (name == null) {
- return defaultValue;
- }
- String value = props.get(name.trim());
- return Long.parseLong(value);
- }
-
- public boolean getBoolean(String name, boolean defaultValue) {
- if (name == null) {
- return defaultValue;
- }
- return Boolean.valueOf(props.get(name.trim()));
- }
-
- @Override
- public boolean equals(java.lang.Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ConfigFile configFile = (ConfigFile) o;
- return Objects.equals(this.type, configFile.type)
- && Objects.equals(this.destFile, configFile.destFile)
- && Objects.equals(this.srcFile, configFile.srcFile);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(type, destFile, srcFile, props);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("class ConfigFile {\n");
-
- sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" destFile: ").append(toIndentedString(destFile)).append("\n");
- sb.append(" srcFile: ").append(toIndentedString(srcFile)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
- sb.append("}");
- return sb.toString();
- }
-
- /**
- * Convert the given object to string with each line indented by 4 spaces
- * (except the first line).
- */
- private String toIndentedString(java.lang.Object o) {
- if (o == null) {
- return "null";
- }
- return o.toString().replace("\n", "\n ");
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
new file mode 100644
index 0000000..7b22e3e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Support for marshalling objects to and from JSON.
+ * This class is NOT thread safe; it constructs an object mapper
+ * as an instance field.
+ * @param <T>
+ */
+public class JsonSerDeser<T> {
+
+ private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class);
+ private static final String UTF_8 = "UTF-8";
+
+ private final Class<T> classType;
+ private final ObjectMapper mapper;
+
+ /**
+ * Create an instance bound to a specific type
+ * @param classType class type
+ */
+ public JsonSerDeser(Class<T> classType) {
+ this.classType = classType;
+ this.mapper = new ObjectMapper();
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ public JsonSerDeser(Class<T> classType, PropertyNamingStrategy namingStrategy) {
+ this(classType);
+ mapper.setPropertyNamingStrategy(namingStrategy);
+ }
+
+ /**
+ * Convert from JSON
+ * @param json input
+ * @return the parsed JSON
+ * @throws IOException IO
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromJson(String json)
+ throws IOException, JsonParseException, JsonMappingException {
+ try {
+ return mapper.readValue(json, classType);
+ } catch (IOException e) {
+ log.error("Exception while parsing json : " + e + "\n" + json, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from a JSON file
+ * @param jsonFile input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromFile(File jsonFile)
+ throws IOException, JsonParseException, JsonMappingException {
+ File absoluteFile = jsonFile.getAbsoluteFile();
+ try {
+ return mapper.readValue(absoluteFile, classType);
+ } catch (IOException e) {
+ log.error("Exception while parsing json file {}", absoluteFile, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from a JSON file
+ * @param resource input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromResource(String resource)
+ throws IOException, JsonParseException, JsonMappingException {
+ try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
+ if (resStream == null) {
+ throw new FileNotFoundException(resource);
+ }
+ return (T) (mapper.readValue(resStream, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json resource {}", resource, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from an input stream, closing the stream afterwards.
+ * @param stream
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ */
+ public T fromStream(InputStream stream) throws IOException {
+ try {
+ return (T) (mapper.readValue(stream, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json input stream", e);
+ throw e;
+ } finally {
+ IOUtils.closeStream(stream);
+ }
+ }
+
+ /**
+ * clone by converting to JSON and back again.
+ * This is much less efficient than any Java clone process.
+ * @param instance instance to duplicate
+ * @return a new instance
+ * @throws IOException problems.
+ */
+ public T fromInstance(T instance) throws IOException {
+ return fromJson(toJson(instance));
+ }
+
+ /**
+ * Deserialize from a byte array
+ * @param b
+ * @return the deserialized value
+ * @throws IOException parse problems
+ */
+ public T fromBytes(byte[] b) throws IOException {
+ String json = new String(b, 0, b.length, UTF_8);
+ return fromJson(json);
+ }
+
+ /**
+ * Load from a Hadoop filesystem
+ * @param fs filesystem
+ * @param path path
+ * @return a loaded CD
+ * @throws IOException IO problems
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public T load(FileSystem fs, Path path)
+ throws IOException, JsonParseException, JsonMappingException {
+ FileStatus status = fs.getFileStatus(path);
+ long len = status.getLen();
+ byte[] b = new byte[(int) len];
+ FSDataInputStream dataInputStream = fs.open(path);
+ int count = dataInputStream.read(b);
+ if (count != len) {
+ throw new EOFException("Read of " + path +" finished prematurely");
+ }
+ return fromBytes(b);
+ }
+
+
+ /**
+ * Save to a hadoop filesystem
+ * @param fs filesystem
+ * @param path path
+ * @param instance instance to save
+ * @param overwrite should any existing file be overwritten
+ * @throws IOException IO exception
+ */
+ public void save(FileSystem fs, Path path, T instance,
+ boolean overwrite) throws
+ IOException {
+ FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
+ writeJsonAsBytes(instance, dataOutputStream);
+ }
+
+ /**
+ * Save an instance to a file
+ * @param instance instance to save
+ * @param file file
+ * @throws IOException
+ */
+ public void save(T instance, File file) throws
+ IOException {
+ writeJsonAsBytes(instance, new FileOutputStream(file.getAbsoluteFile()));
+ }
+
+ /**
+ * Write the json as bytes -then close the file
+ * @param dataOutputStream an outout stream that will always be closed
+ * @throws IOException on any failure
+ */
+ private void writeJsonAsBytes(T instance,
+ OutputStream dataOutputStream) throws IOException {
+ try {
+ String json = toJson(instance);
+ byte[] b = json.getBytes(UTF_8);
+ dataOutputStream.write(b);
+ dataOutputStream.flush();
+ dataOutputStream.close();
+ } finally {
+ IOUtils.closeStream(dataOutputStream);
+ }
+ }
+
+ /**
+ * Convert an object to a JSON string
+ * @param instance instance to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public String toJson(T instance) throws IOException,
+ JsonGenerationException,
+ JsonMappingException {
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ return mapper.writeValueAsString(instance);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
new file mode 100644
index 0000000..c0712c3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
@@ -0,0 +1,680 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslPropertiesResolver;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.Cipher;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.InvocationTargetException;
+import java.net.InetAddress;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.security.UserGroupInformation.*;
+import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+
+/**
+ * Kerberos diagnostics
+ * At some point this may move to hadoop core, so please keep use of slider
+ * methods and classes to ~0.
+ *
+ * This operation expands some of the diagnostic output of the security code,
+ * but not all. For completeness
+ *
+ * Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
+ * Set the log level for {@code org.apache.hadoop.security=DEBUG}
+ */
+public class KerberosDiags implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class);
+ public static final String KRB5_CCNAME = "KRB5CCNAME";
+ public static final String JAVA_SECURITY_KRB5_CONF
+ = "java.security.krb5.conf";
+ public static final String JAVA_SECURITY_KRB5_REALM
+ = "java.security.krb5.realm";
+ public static final String SUN_SECURITY_KRB5_DEBUG
+ = "sun.security.krb5.debug";
+ public static final String SUN_SECURITY_SPNEGO_DEBUG
+ = "sun.security.spnego.debug";
+ public static final String SUN_SECURITY_JAAS_FILE
+ = "java.security.auth.login.config";
+ public static final String KERBEROS_KINIT_COMMAND
+ = "hadoop.kerberos.kinit.command";
+ public static final String HADOOP_AUTHENTICATION_IS_DISABLED
+ = "Hadoop authentication is disabled";
+ public static final String UNSET = "(unset)";
+ public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
+
+ private final Configuration conf;
+ private final List<String> services;
+ private final PrintStream out;
+ private final File keytab;
+ private final String principal;
+ private final long minKeyLength;
+ private final boolean securityRequired;
+
+ public static final String CAT_JVM = "JVM";
+ public static final String CAT_JAAS = "JAAS";
+ public static final String CAT_CONFIG = "CONFIG";
+ public static final String CAT_LOGIN = "LOGIN";
+ public static final String CAT_KERBEROS = "KERBEROS";
+ public static final String CAT_SASL = "SASL";
+
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+ public KerberosDiags(Configuration conf,
+ PrintStream out,
+ List<String> services,
+ File keytab,
+ String principal,
+ long minKeyLength,
+ boolean securityRequired) {
+ this.conf = conf;
+ this.services = services;
+ this.keytab = keytab;
+ this.principal = principal;
+ this.out = out;
+ this.minKeyLength = minKeyLength;
+ this.securityRequired = securityRequired;
+ }
+
+ @Override
+ public void close() throws IOException {
+ flush();
+ }
+
+ /**
+ * Execute diagnostics.
+ * <p>
+ * Things it would be nice if UGI made accessible
+ * <ol>
+ * <li>A way to enable JAAS debug programatically</li>
+ * <li>Access to the TGT</li>
+ * </ol>
+ * @return true if security was enabled and all probes were successful
+ * @throws KerberosDiagsFailure explicitly raised failure
+ * @throws Exception other security problems
+ */
+ @SuppressWarnings("deprecation")
+ public boolean execute() throws Exception {
+
+ title("Kerberos Diagnostics scan at %s",
+ new Date(System.currentTimeMillis()));
+
+ // check that the machine has a name
+ println("Hostname: %s",
+ InetAddress.getLocalHost().getCanonicalHostName());
+
+ // Fail fast on a JVM without JCE installed.
+ validateKeyLength();
+
+ // look at realm
+ println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName());
+ printDefaultRealm();
+
+ title("System Properties");
+ for (String prop : new String[]{
+ JAVA_SECURITY_KRB5_CONF,
+ JAVA_SECURITY_KRB5_REALM,
+ SUN_SECURITY_KRB5_DEBUG,
+ SUN_SECURITY_SPNEGO_DEBUG,
+ SUN_SECURITY_JAAS_FILE
+ }) {
+ printSysprop(prop);
+ }
+
+ title("Environment Variables");
+ for (String env : new String[]{
+ "HADOOP_JAAS_DEBUG",
+ KRB5_CCNAME,
+ "HADOOP_USER_NAME",
+ "HADOOP_PROXY_USER",
+ HADOOP_TOKEN_FILE_LOCATION,
+ }) {
+ printEnv(env);
+ }
+
+ for (String prop : new String[]{
+ KERBEROS_KINIT_COMMAND,
+ HADOOP_SECURITY_AUTHENTICATION,
+ HADOOP_SECURITY_AUTHORIZATION,
+ "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6
+ "hadoop.security.dns.interface", // not in 2.6
+ "hadoop.security.dns.nameserver", // not in 2.6
+ HADOOP_RPC_PROTECTION,
+ HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
+ HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX,
+ HADOOP_SECURITY_GROUP_MAPPING,
+ "hadoop.security.impersonation.provider.class", // not in 2.6
+ "dfs.data.transfer.protection" // HDFS
+ }) {
+ printConfOpt(prop);
+ }
+
+ // check that authentication is enabled
+ if (SecurityUtil.getAuthenticationMethod(conf)
+ .equals(AuthenticationMethod.SIMPLE)) {
+ println(HADOOP_AUTHENTICATION_IS_DISABLED);
+ failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED);
+ // no security, skip rest of test
+ return false;
+ }
+
+ validateKrb5File();
+ validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS);
+ validateSasl("dfs.data.transfer.saslproperties.resolver.class");
+ validateKinitExecutable();
+ validateJAAS();
+ // now the big test: login, then try again
+ boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG);
+ boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG);
+ try {
+ title("Logging in");
+
+ if (keytab != null) {
+ dumpKeytab(keytab);
+ loginFromKeytab();
+ } else {
+ UserGroupInformation loginUser = getLoginUser();
+ dumpUGI("Log in user", loginUser);
+ validateUGI("Login user", loginUser);
+ println("Ticket based login: %b", isLoginTicketBased());
+ println("Keytab based login: %b", isLoginKeytabBased());
+ }
+
+ return true;
+ } finally {
+ // restore original system properties
+ System.setProperty(SUN_SECURITY_KRB5_DEBUG,
+ Boolean.toString(krb5Debug));
+ System.setProperty(SUN_SECURITY_SPNEGO_DEBUG,
+ Boolean.toString(spnegoDebug));
+ }
+ }
+
+ /**
+ * Fail fast on a JVM without JCE installed.
+ *
+ * This is a recurrent problem
+ * (that is: it keeps creeping back with JVM updates);
+ * a fast failure is the best tactic
+ * @throws NoSuchAlgorithmException
+ */
+
+ protected void validateKeyLength() throws NoSuchAlgorithmException {
+ int aesLen = Cipher.getMaxAllowedKeyLength("AES");
+ println("Maximum AES encryption key length %d bits", aesLen);
+ failif (aesLen < minKeyLength,
+ CAT_JVM,
+ "Java Cryptography Extensions are not installed on this JVM."
+ +" Maximum supported key length %s - minimum required %d",
+ aesLen, minKeyLength);
+ }
+
+ /**
+ * Get the default realm.
+ * <p>
+ * Not having a default realm may be harmless, so is noted at info.
+ * All other invocation failures are downgraded to warn, as
+ * follow-on actions may still work.
+ * failure to invoke the method via introspection is rejected,
+ * as it's a sign of JVM compatibility issues that may have other
+ * consequences
+ */
+ protected void printDefaultRealm() {
+ try {
+ println("Default Realm = %s",
+ getDefaultRealm());
+ } catch (ClassNotFoundException
+ | IllegalAccessException
+ | NoSuchMethodException e) {
+
+ throw new KerberosDiagsFailure(CAT_JVM, e,
+ "Failed to invoke krb5.Config.getDefaultRealm: %s", e);
+ } catch (InvocationTargetException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+ if (cause.toString().contains(NO_DEFAULT_REALM)) {
+ // exception raised if there is no default realm. This is not
+ // always a problem, so downgrade to a message.
+ println("Host has no default realm");
+ LOG.debug(cause.toString(), cause);
+ } else {
+ println("Kerberos.getDefaultRealm() failed: %s\n%s",
+ cause,
+ org.apache.hadoop.util.StringUtils.stringifyException(cause));
+ }
+ }
+ }
+
+ /**
+ * Locate the krb5.conf file and dump it.
+ * No-op on windows.
+ * @throws IOException
+ */
+ private void validateKrb5File() throws IOException {
+ if (!Shell.WINDOWS) {
+ title("Locating Kerberos configuration file");
+ String krbPath = "/etc/krb5.conf";
+ String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF);
+ if (jvmKrbPath != null) {
+ println("Setting kerberos path from sysprop %s: %s",
+ JAVA_SECURITY_KRB5_CONF, jvmKrbPath);
+ krbPath = jvmKrbPath;
+ }
+
+ String krb5name = System.getenv(KRB5_CCNAME);
+ if (krb5name != null) {
+ println("Setting kerberos path from environment variable %s: %s",
+ KRB5_CCNAME, krb5name);
+ krbPath = krb5name;
+ if (jvmKrbPath != null) {
+ println("Warning - both %s and %s were set - %s takes priority",
+ JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME);
+ }
+ }
+
+ File krbFile = new File(krbPath);
+ println("Kerberos configuration file = %s", krbFile);
+ failif(!krbFile.exists(),
+ CAT_KERBEROS,
+ "Kerberos configuration file %s not found", krbFile);
+ dump(krbFile);
+ }
+ }
+
+ /**
+ * Dump a keytab: list all principals.
+ * @param keytabFile the keytab file
+ * @throws IOException IO problems
+ */
+ public void dumpKeytab(File keytabFile) throws IOException {
+ title("Examining keytab %s", keytabFile);
+ File kt = keytabFile.getCanonicalFile();
+ failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
+ failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt);
+
+ String[] names = getPrincipalNames(keytabFile.getCanonicalPath(),
+ Pattern.compile(".*"));
+ println("keytab entry count: %d", names.length);
+ for (String name : names) {
+ println(" %s", name);
+ }
+ println("-----");
+ }
+
+ /**
+ * Log in from a keytab, dump the UGI, validate it, then try and log in again.
+ * That second-time login catches JVM/Hadoop compatibility problems.
+ * @throws IOException
+ */
+ private void loginFromKeytab() throws IOException {
+ UserGroupInformation ugi;
+ String identity;
+ if (keytab != null) {
+ File kt = keytab.getCanonicalFile();
+ println("Using keytab %s principal %s", kt, principal);
+ identity = principal;
+
+ failif(StringUtils.isEmpty(principal), CAT_KERBEROS,
+ "No principal defined");
+ ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath());
+ dumpUGI(identity, ugi);
+ validateUGI(principal, ugi);
+
+ title("Attempting to log in from keytab again");
+ // package scoped -hence the reason why this class must be in the
+ // hadoop.security package
+ setShouldRenewImmediatelyForTests(true);
+ // attempt a new login
+ ugi.reloginFromKeytab();
+ } else {
+ println("No keytab: logging is as current user");
+ }
+ }
+
+ /**
+ * Dump a UGI.
+ * @param title title of this section
+ * @param ugi UGI to dump
+ * @throws IOException
+ */
+ private void dumpUGI(String title, UserGroupInformation ugi)
+ throws IOException {
+ title(title);
+ println("UGI instance = %s", ugi);
+ println("Has kerberos credentials: %b", ugi.hasKerberosCredentials());
+ println("Authentication method: %s", ugi.getAuthenticationMethod());
+ println("Real Authentication method: %s",
+ ugi.getRealAuthenticationMethod());
+ title("Group names");
+ for (String name : ugi.getGroupNames()) {
+ println(name);
+ }
+ title("Credentials");
+ Credentials credentials = ugi.getCredentials();
+ List<Text> secretKeys = credentials.getAllSecretKeys();
+ title("Secret keys");
+ if (!secretKeys.isEmpty()) {
+ for (Text secret: secretKeys) {
+ println("%s", secret);
+ }
+ } else {
+ println("(none)");
+ }
+
+ dumpTokens(ugi);
+ }
+
+ /**
+ * Validate the UGI: verify it is kerberized.
+ * @param messagePrefix message in exceptions
+ * @param user user to validate
+ */
+ private void validateUGI(String messagePrefix, UserGroupInformation user) {
+ failif(!user.hasKerberosCredentials(),
+ CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user);
+ failif(user.getAuthenticationMethod() == null,
+ CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user);
+ }
+
+ /**
+ * A cursory look at the {@code kinit} executable.
+ * If it is an absolute path: it must exist with a size > 0.
+ * If it is just a command, it has to be on the path. There's no check
+ * for that -but the PATH is printed out.
+ */
+ private void validateKinitExecutable() {
+ String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, "");
+ if (!kinit.isEmpty()) {
+ File kinitPath = new File(kinit);
+ println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath);
+ if (kinitPath.isAbsolute()) {
+ failif(!kinitPath.exists(), CAT_KERBEROS,
+ "%s executable does not exist: %s",
+ KERBEROS_KINIT_COMMAND, kinitPath);
+ failif(!kinitPath.isFile(), CAT_KERBEROS,
+ "%s path does not refer to a file: %s",
+ KERBEROS_KINIT_COMMAND, kinitPath);
+ failif(kinitPath.length() == 0, CAT_KERBEROS,
+ "%s file is empty: %s",
+ KERBEROS_KINIT_COMMAND, kinitPath);
+ } else {
+ println("Executable %s is relative -must be on the PATH", kinit);
+ printEnv("PATH");
+ }
+ }
+ }
+
+ /**
+ * Try to load the SASL resolver.
+ * @param saslPropsResolverKey key for the SASL resolver
+ */
+ private void validateSasl(String saslPropsResolverKey) {
+ title("Resolving SASL property %s", saslPropsResolverKey);
+ String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey);
+ try {
+ Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
+ saslPropsResolverKey,
+ SaslPropertiesResolver.class, SaslPropertiesResolver.class);
+ println("Resolver is %s", resolverClass);
+ } catch (RuntimeException e) {
+ throw new KerberosDiagsFailure(CAT_SASL, e,
+ "Failed to load %s class %s",
+ saslPropsResolverKey, saslPropsResolver);
+ }
+ }
+
+ /**
+ * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE}
+ * property.
+ */
+ private void validateJAAS() {
+ String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE);
+ if (jaasFilename != null) {
+ title("JAAS");
+ File jaasFile = new File(jaasFilename);
+ println("JAAS file is defined in %s: %s",
+ SUN_SECURITY_JAAS_FILE, jaasFile);
+ failif(!jaasFile.exists(), CAT_JAAS,
+ "JAAS file does not exist: %s", jaasFile);
+ failif(!jaasFile.isFile(), CAT_JAAS,
+ "Specified JAAS file is not a file: %s", jaasFile);
+ }
+ }
+
+ /**
+ * Dump all tokens of a user
+ * @param user user
+ */
+ public void dumpTokens(UserGroupInformation user) {
+ Collection<Token<? extends TokenIdentifier>> tokens
+ = user.getCredentials().getAllTokens();
+ title("Token Count: %d", tokens.size());
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ println("Token %s", token.getKind());
+ }
+ }
+
+ /**
+ * Set the System property to true; return the old value for caching
+ * @param sysprop property
+ * @return the previous value
+ */
+ private boolean getAndSet(String sysprop) {
+ boolean old = Boolean.getBoolean(sysprop);
+ System.setProperty(sysprop, "true");
+ return old;
+ }
+
+ /**
+ * Flush all active output channels, including {@Code System.err},
+ * so as to stay in sync with any JRE log messages.
+ */
+ private void flush() {
+ if (out != null) {
+ out.flush();
+ } else {
+ System.out.flush();
+ }
+ System.err.flush();
+ }
+
+ /**
+ * Format and print a line of output.
+ * This goes to any output file, or
+ * is logged at info. The output is flushed before and after, to
+ * try and stay in sync with JRE logging.
+ * @param format format string
+ * @param args any arguments
+ */
+ @VisibleForTesting
+ public void println(String format, Object... args) {
+ println(format(format, args));
+ }
+
+ /**
+ * Print a line of output. This goes to any output file, or
+ * is logged at info. The output is flushed before and after, to
+ * try and stay in sync with JRE logging.
+ * @param msg message string
+ */
+ @VisibleForTesting
+ private void println(String msg) {
+ flush();
+ if (out != null) {
+ out.println(msg);
+ } else {
+ LOG.info(msg);
+ }
+ flush();
+ }
+
+ /**
+ * Print a title entry
+ * @param format format string
+ * @param args any arguments
+ */
+ private void title(String format, Object... args) {
+ println("");
+ println("");
+ String msg = "== " + format(format, args) + " ==";
+ println(msg);
+ println("");
+ }
+
+ /**
+ * Print a system property, or {@link #UNSET} if unset.
+ * @param property property to print
+ */
+ private void printSysprop(String property) {
+ println("%s = \"%s\"", property,
+ System.getProperty(property, UNSET));
+ }
+
+ /**
+ * Print a configuration option, or {@link #UNSET} if unset.
+ * @param option option to print
+ */
+ private void printConfOpt(String option) {
+ println("%s = \"%s\"", option, conf.get(option, UNSET));
+ }
+
+ /**
+ * Print an environment variable's name and value; printing
+ * {@link #UNSET} if it is not set
+ * @param variable environment variable
+ */
+ private void printEnv(String variable) {
+ String env = System.getenv(variable);
+ println("%s = \"%s\"", variable, env != null ? env : UNSET);
+ }
+
+ /**
+ * Dump any file to standard out; add a trailing newline
+ * @param file file to dump
+ * @throws IOException IO problems
+ */
+ public void dump(File file) throws IOException {
+ try (FileInputStream in = new FileInputStream(file)) {
+ for (String line : IOUtils.readLines(in)) {
+ println("%s", line);
+ }
+ }
+ println("");
+ }
+
+ /**
+ * Format and raise a failure
+ *
+ * @param category category for exception
+ * @param message string formatting message
+ * @param args any arguments for the formatting
+ * @throws KerberosDiagsFailure containing the formatted text
+ */
+ private void fail(String category, String message, Object... args)
+ throws KerberosDiagsFailure {
+ throw new KerberosDiagsFailure(category, message, args);
+ }
+
+ /**
+ * Conditional failure with string formatted arguments
+ * @param condition failure condition
+ * @param category category for exception
+ * @param message string formatting message
+ * @param args any arguments for the formatting
+ * @throws KerberosDiagsFailure containing the formatted text
+ * if the condition was met
+ */
+ private void failif(boolean condition,
+ String category,
+ String message,
+ Object... args)
+ throws KerberosDiagsFailure {
+ if (condition) {
+ fail(category, message, args);
+ }
+ }
+
+ /**
+ * Format a string, treating a call where there are no varags values
+ * as a string to pass through unformatted.
+ * @param message message, which is either a format string + args, or
+ * a general string
+ * @param args argument array
+ * @return a string for printing.
+ */
+ public static String format(String message, Object... args) {
+ if (args.length == 0) {
+ return message;
+ } else {
+ return String.format(message, args);
+ }
+ }
+
+ /**
+ * Diagnostics failures return the exit code 41, "unauthorized".
+ *
+ * They have a category, initially for testing: the category can be
+ * validated without having to match on the entire string.
+ */
+ public static class KerberosDiagsFailure extends ExitUtil.ExitException {
+ private final String category;
+
+ public KerberosDiagsFailure(String category, String message) {
+ super(41, category + ": " + message);
+ this.category = category;
+ }
+
+ public KerberosDiagsFailure(String category, String message, Object... args) {
+ this(category, format(message, args));
+ }
+
+ public KerberosDiagsFailure(String category, Throwable throwable,
+ String message, Object... args) {
+ this(category, message, args);
+ initCause(throwable);
+ }
+
+ public String getCategory() {
+ return category;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
new file mode 100644
index 0000000..108ca22
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import java.util.regex.Pattern;
+
+/**
+ * Utility class to validate strings against a predefined pattern.
+ */
+public class PatternValidator {
+
+ public static final String E_INVALID_NAME =
+ "Invalid name %s does not match the pattern %s ";
+ private final Pattern valid;
+ private final String pattern;
+
+ public PatternValidator(String pattern) {
+ this.pattern = pattern;
+ valid = Pattern.compile(pattern);
+ }
+
+ /**
+ * Validate the name -restricting it to the set defined in
+ * @param name name to validate
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public void validate(String name) {
+ if (!matches(name)) {
+ throw new IllegalArgumentException(
+ String.format(E_INVALID_NAME, name, pattern));
+ }
+ }
+
+ /**
+ * Query to see if the pattern matches
+ * @param name name to validate
+ * @return true if the string matches the pattern
+ */
+ public boolean matches(String name) {
+ return valid.matcher(name).matches();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
new file mode 100644
index 0000000..2dbf37f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * a scanner which can take an input string for a range or scan the lot.
+ */
+public class PortScanner {
+ private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
+ private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$");
+
+ private List<Integer> remainingPortsToCheck;
+
+ public PortScanner() {
+ }
+
+ public void setPortRange(String input) throws BadConfigException {
+ // first split based on commas
+ Set<Integer> inputPorts= new TreeSet<Integer>();
+ String[] ranges = input.split(",");
+ for ( String range : ranges ) {
+ if (range.trim().isEmpty()) {
+ continue;
+ }
+ Matcher m = SINGLE_NUMBER.matcher(range.trim());
+ if (m.find()) {
+ inputPorts.add(Integer.parseInt(m.group()));
+ continue;
+ }
+ m = NUMBER_RANGE.matcher(range.trim());
+ if (m.find()) {
+ String[] boundaryValues = m.group(0).split("-");
+ int start = Integer.parseInt(boundaryValues[0].trim());
+ int end = Integer.parseInt(boundaryValues[1].trim());
+ if (end < start) {
+ throw new BadConfigException("End of port range is before start: "
+ + range + " in input: " + input);
+ }
+ for (int i = start; i < end + 1; i++) {
+ inputPorts.add(i);
+ }
+ continue;
+ }
+ throw new BadConfigException("Bad port range: " + range + " in input: "
+ + input);
+ }
+ if (inputPorts.size() == 0) {
+ throw new BadConfigException("No ports found in range: " + input);
+ }
+ this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
+ }
+
+ public List<Integer> getRemainingPortsToCheck() {
+ return remainingPortsToCheck;
+ }
+
+ public int getAvailablePort() throws SliderException, IOException {
+ if (remainingPortsToCheck != null) {
+ return getAvailablePortViaPortArray();
+ } else {
+ return SliderUtils.getOpenPort();
+ }
+ }
+
+ private int getAvailablePortViaPortArray() throws SliderException {
+ boolean found = false;
+ int availablePort = -1;
+ Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
+ while (portsToCheck.hasNext() && !found) {
+ int portToCheck = portsToCheck.next();
+ found = SliderUtils.isPortAvailable(portToCheck);
+ if (found) {
+ availablePort = portToCheck;
+ portsToCheck.remove();
+ }
+ }
+
+ if (availablePort < 0) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "No available ports found in configured range {}",
+ remainingPortsToCheck);
+ }
+
+ return availablePort;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
new file mode 100644
index 0000000..9d00b3c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have
+ * to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedConfiguration {
+
+ public String description;
+ public long updated;
+
+ public String updatedTime;
+
+ public Map<String, String> entries = new HashMap<>();
+
+ public PublishedConfiguration() {
+ }
+
+ /**
+ * build an empty published configuration
+ * @param description configuration description
+ */
+ public PublishedConfiguration(String description) {
+ this.description = description;
+ }
+
+ /**
+ * Build a configuration from the entries
+ * @param description configuration description
+ * @param entries entries to put
+ */
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> entries) {
+ this.description = description;
+ putValues(entries);
+ }
+
+ /**
+ * Build a published configuration, using the keys from keysource,
+ * but resolving the values from the value source, via Configuration.get()
+ * @param description configuration description
+ * @param keysource source of keys
+ * @param valuesource source of values
+ */
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> keysource,
+ Configuration valuesource) {
+ this.description = description;
+ putValues(ConfigHelper.resolveConfiguration(keysource, valuesource));
+ }
+
+
+ /**
+ * Is the configuration empty. This means either that it has not
+ * been given any values, or it is stripped down copy set down over the
+ * wire.
+ * @return true if it is empty
+ */
+ public boolean isEmpty() {
+ return entries.isEmpty();
+ }
+
+
+ public void setUpdated(long updated) {
+ this.updated = updated;
+ this.updatedTime = new Date(updated).toString();
+ }
+
+ public long getUpdated() {
+ return updated;
+ }
+
+ /**
+ * Set the values from an iterable (this includes a Hadoop Configuration
+ * and Java properties object).
+ * Any existing value set is discarded
+ * @param entries entries to put
+ */
+ public void putValues(Iterable<Map.Entry<String, String>> entries) {
+ this.entries = new HashMap<String, String>();
+ for (Map.Entry<String, String> entry : entries) {
+ this.entries.put(entry.getKey(), entry.getValue());
+ }
+
+ }
+
+ /**
+ * Convert to Hadoop XML
+ * @return the configuration as a Hadoop Configuratin
+ */
+ public Configuration asConfiguration() {
+ Configuration conf = new Configuration(false);
+ try {
+ ConfigHelper.addConfigMap(conf, entries, "");
+ } catch (BadConfigException e) {
+ // triggered on a null value; switch to a runtime (and discard the stack)
+ throw new RuntimeException(e.toString());
+ }
+ return conf;
+ }
+
+ public String asConfigurationXML() throws IOException {
+ return ConfigHelper.toXml(asConfiguration());
+ }
+
+ /**
+ * Convert values to properties
+ * @return a property file
+ */
+ public Properties asProperties() {
+ Properties props = new Properties();
+ props.putAll(entries);
+ return props;
+ }
+
+ /**
+ * Return the values as json string
+ * @return the JSON representation
+ * @throws IOException marshalling failure
+ */
+ public String asJson() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ String json = mapper.writeValueAsString(entries);
+ return json;
+ }
+
+
+ /**
+ * This makes a copy without the nested content -so is suitable
+ * for returning as part of the list of a parent's values
+ * @return the copy
+ */
+ public PublishedConfiguration shallowCopy() {
+ PublishedConfiguration that = new PublishedConfiguration();
+ that.description = this.description;
+ that.updated = this.updated;
+ that.updatedTime = this.updatedTime;
+ return that;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("PublishedConfiguration{");
+ sb.append("description='").append(description).append('\'');
+ sb.append(" entries = ").append(entries.size());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Create an outputter for a given format
+ * @param format format to use
+ * @return an instance of output
+ */
+ public PublishedConfigurationOutputter createOutputter(ConfigFormat format) {
+ return PublishedConfigurationOutputter.createOutputter(format, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
new file mode 100644
index 0000000..88ecf2c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.DumperOptions.FlowStyle;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.util.Properties;
+
+/**
+ * Output a published configuration
+ */
+public abstract class PublishedConfigurationOutputter {
+
+ private static final String COMMENTS = "Generated by Apache Slider";
+
+ protected final PublishedConfiguration owner;
+
+ protected PublishedConfigurationOutputter(PublishedConfiguration owner) {
+ this.owner = owner;
+ }
+
+ /**
+ * Save the config to a destination file, in the format of this outputter
+ * @param dest destination file
+ * @throws IOException
+ */
+/* JDK7
+ public void save(File dest) throws IOException {
+ try(FileOutputStream out = new FileOutputStream(dest)) {
+ save(out);
+ out.close();
+ }
+ }
+*/
+ public void save(File dest) throws IOException {
+ FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
+ }
+
+ /**
+ * Save the content. The default saves the asString() value
+ * to the output stream
+ * @param out output stream
+ * @throws IOException
+ */
+ public void save(OutputStream out) throws IOException {
+ IOUtils.write(asString(), out, Charsets.UTF_8);
+ }
+ /**
+ * Convert to a string
+ * @return the string form
+ * @throws IOException
+ */
+ public abstract String asString() throws IOException;
+
+ /**
+ * Create an outputter for the chosen format
+ * @param format format enumeration
+ * @param owner owning config
+ * @return the outputter
+ */
+
+ public static PublishedConfigurationOutputter createOutputter(ConfigFormat format,
+ PublishedConfiguration owner) {
+ Preconditions.checkNotNull(owner);
+ switch (format) {
+ case XML:
+ case HADOOP_XML:
+ return new XmlOutputter(owner);
+ case PROPERTIES:
+ return new PropertiesOutputter(owner);
+ case JSON:
+ return new JsonOutputter(owner);
+ case ENV:
+ return new EnvOutputter(owner);
+ case TEMPLATE:
+ return new TemplateOutputter(owner);
+ case YAML:
+ return new YamlOutputter(owner);
+ default:
+ throw new RuntimeException("Unsupported format :" + format);
+ }
+ }
+
+ public static class XmlOutputter extends PublishedConfigurationOutputter {
+
+
+ private final Configuration configuration;
+
+ public XmlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ configuration = owner.asConfiguration();
+ }
+
+ @Override
+ public void save(OutputStream out) throws IOException {
+ configuration.writeXml(out);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return ConfigHelper.toXml(configuration);
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+ }
+
+ public static class PropertiesOutputter extends PublishedConfigurationOutputter {
+
+ private final Properties properties;
+
+ public PropertiesOutputter(PublishedConfiguration owner) {
+ super(owner);
+ properties = owner.asProperties();
+ }
+
+ @Override
+ public void save(OutputStream out) throws IOException {
+ properties.store(out, COMMENTS);
+ }
+
+
+ public String asString() throws IOException {
+ StringWriter sw = new StringWriter();
+ properties.store(sw, COMMENTS);
+ return sw.toString();
+ }
+ }
+
+
+ public static class JsonOutputter extends PublishedConfigurationOutputter {
+
+ public JsonOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return owner.asJson();
+ }
+ }
+
+
+ public static class EnvOutputter extends PublishedConfigurationOutputter {
+
+ public EnvOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ if (!owner.entries.containsKey("content")) {
+ throw new IOException("Configuration has no content field and cannot " +
+ "be retrieved as type 'env'");
+ }
+ String content = owner.entries.get("content");
+ return ConfigUtils.replaceProps(owner.entries, content);
+ }
+ }
+
+ public static class TemplateOutputter extends EnvOutputter {
+ public TemplateOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+ }
+
+ public static class YamlOutputter extends PublishedConfigurationOutputter {
+
+ private final Yaml yaml;
+
+ public YamlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(FlowStyle.BLOCK);
+ yaml = new Yaml(options);
+ }
+
+ public String asString() throws IOException {
+ return yaml.dump(owner.entries);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
new file mode 100644
index 0000000..140204a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+
+/**
+ * Serialized form of an service report which can be persisted
+ * and then parsed. It can not be converted back into a
+ * real YARN service report
+ *
+ * Useful for testing
+ */
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class SerializedApplicationReport {
+
+ public String applicationId;
+ public String applicationAttemptId;
+ public String name;
+ public String applicationType;
+ public String user;
+ public String queue;
+ public String host;
+ public Integer rpcPort;
+ public String state;
+ public String diagnostics;
+ public String url;
+ /**
+ * This value is non-null only when a report is generated from a submission context.
+ * The YARN {@link ApplicationReport} structure does not propagate this value
+ * from the RM.
+ */
+ public Long submitTime;
+ public Long startTime;
+ public Long finishTime;
+ public String finalStatus;
+ public String origTrackingUrl;
+ public Float progress;
+
+ public SerializedApplicationReport() {
+ }
+
+ public SerializedApplicationReport(ApplicationReport report) {
+ this.applicationId = report.getApplicationId().toString();
+ ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
+ this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A";
+ this.name = report.getName();
+ this.applicationType = report.getApplicationType();
+ this.user = report.getUser();
+ this.queue = report.getQueue();
+ this.host = report.getHost();
+ this.rpcPort = report.getRpcPort();
+ this.state = report.getYarnApplicationState().toString();
+ this.diagnostics = report.getDiagnostics();
+ this.startTime = report.getStartTime();
+ this.finishTime = report.getFinishTime();
+ FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
+ this.finalStatus = appStatus == null ? "" : appStatus.toString();
+ this.progress = report.getProgress();
+ this.url = report.getTrackingUrl();
+ this.origTrackingUrl= report.getOriginalTrackingUrl();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return ApplicationReportSerDeser.toString(this);
+ } catch (IOException e) {
+ return super.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
new file mode 100644
index 0000000..de82580
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
+import org.apache.hadoop.yarn.service.conf.RestApiConstants;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class ServiceApiUtil {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceApiUtil.class);
+ public static JsonSerDeser<Service> jsonSerDeser =
+ new JsonSerDeser<>(Service.class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+ private static final PatternValidator namePattern
+ = new PatternValidator("[a-z][a-z0-9-]*");
+
+ @VisibleForTesting
+ public static void setJsonSerDeser(JsonSerDeser jsd) {
+ jsonSerDeser = jsd;
+ }
+
+ @VisibleForTesting
+ public static void validateAndResolveService(Service service,
+ SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf) throws
+ IOException {
+ boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
+ RegistryConstants.DEFAULT_DNS_ENABLED);
+ if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
+ .MAX_FQDN_LABEL_LENGTH) {
+ throw new IllegalArgumentException(RestApiErrorMessages
+ .ERROR_USER_NAME_INVALID);
+ }
+ if (StringUtils.isEmpty(service.getName())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
+ }
+
+ validateNameFormat(service.getName(), conf);
+
+ // If the service has no components do top-level checks
+ if (!hasComponent(service)) {
+ // If artifact is of type SERVICE, read other service components
+ if (service.getArtifact() != null && service.getArtifact()
+ .getType() == Artifact.TypeEnum.SERVICE) {
+ if (StringUtils.isEmpty(service.getArtifact().getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ Service otherService = loadService(fs,
+ service.getArtifact().getId());
+ service.setComponents(otherService.getComponents());
+ service.setArtifact(null);
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(service.getQuicklinks(),
+ otherService.getQuicklinks());
+ } else {
+ // Since it is a simple service with no components, create a default
+ // component
+ Component comp = createDefaultComponent(service);
+ validateComponent(comp, fs.getFileSystem(), conf);
+ service.getComponents().add(comp);
+ if (service.getLifetime() == null) {
+ service.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
+ }
+ return;
+ }
+ }
+
+ // Validate there are no component name collisions (collisions are not
+ // currently supported) and add any components from external services
+ // TODO allow name collisions? see AppState#roles
+ // TODO or add prefix to external component names?
+ Configuration globalConf = service.getConfiguration();
+ Set<String> componentNames = new HashSet<>();
+ List<Component> componentsToRemove = new ArrayList<>();
+ List<Component> componentsToAdd = new ArrayList<>();
+ for (Component comp : service.getComponents()) {
+ int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
+ maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length();
+ if (dnsEnabled && comp.getName().length() > maxCompLength) {
+ throw new IllegalArgumentException(String.format(RestApiErrorMessages
+ .ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName()));
+ }
+ if (componentNames.contains(comp.getName())) {
+ throw new IllegalArgumentException("Component name collision: " +
+ comp.getName());
+ }
+ // If artifact is of type SERVICE (which cannot be filled from
+ // global), read external service and add its components to this
+ // service
+ if (comp.getArtifact() != null && comp.getArtifact().getType() ==
+ Artifact.TypeEnum.SERVICE) {
+ if (StringUtils.isEmpty(comp.getArtifact().getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ LOG.info("Marking {} for removal", comp.getName());
+ componentsToRemove.add(comp);
+ List<Component> externalComponents = getComponents(fs,
+ comp.getArtifact().getId());
+ for (Component c : externalComponents) {
+ Component override = service.getComponent(c.getName());
+ if (override != null && override.getArtifact() == null) {
+ // allow properties from external components to be overridden /
+ // augmented by properties in this component, except for artifact
+ // which must be read from external component
+ override.mergeFrom(c);
+ LOG.info("Merging external component {} from external {}", c
+ .getName(), comp.getName());
+ } else {
+ if (componentNames.contains(c.getName())) {
+ throw new IllegalArgumentException("Component name collision: " +
+ c.getName());
+ }
+ componentNames.add(c.getName());
+ componentsToAdd.add(c);
+ LOG.info("Adding component {} from external {}", c.getName(),
+ comp.getName());
+ }
+ }
+ } else {
+ // otherwise handle as a normal component
+ componentNames.add(comp.getName());
+ // configuration
+ comp.getConfiguration().mergeFrom(globalConf);
+ }
+ }
+ service.getComponents().removeAll(componentsToRemove);
+ service.getComponents().addAll(componentsToAdd);
+
+ // Validate components and let global values take effect if component level
+ // values are not provided
+ Artifact globalArtifact = service.getArtifact();
+ Resource globalResource = service.getResource();
+ Long globalNumberOfContainers = service.getNumberOfContainers();
+ String globalLaunchCommand = service.getLaunchCommand();
+ for (Component comp : service.getComponents()) {
+ // fill in global artifact unless it is type SERVICE
+ if (comp.getArtifact() == null && service.getArtifact() != null
+ && service.getArtifact().getType() != Artifact.TypeEnum
+ .SERVICE) {
+ comp.setArtifact(globalArtifact);
+ }
+ // fill in global resource
+ if (comp.getResource() == null) {
+ comp.setResource(globalResource);
+ }
+ // fill in global container count
+ if (comp.getNumberOfContainers() == null) {
+ comp.setNumberOfContainers(globalNumberOfContainers);
+ }
+ // fill in global launch command
+ if (comp.getLaunchCommand() == null) {
+ comp.setLaunchCommand(globalLaunchCommand);
+ }
+ // validate dependency existence
+ if (comp.getDependencies() != null) {
+ for (String dependency : comp.getDependencies()) {
+ if (!componentNames.contains(dependency)) {
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency,
+ comp.getName()));
+ }
+ }
+ }
+ validateComponent(comp, fs.getFileSystem(), conf);
+ }
+
+ // validate dependency tree
+ sortByDependencies(service.getComponents());
+
+ // Service lifetime if not specified, is set to unlimited lifetime
+ if (service.getLifetime() == null) {
+ service.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
+ }
+ }
+
+ private static void validateComponent(Component comp, FileSystem fs,
+ org.apache.hadoop.conf.Configuration conf)
+ throws IOException {
+ validateNameFormat(comp.getName(), conf);
+
+ AbstractClientProvider compClientProvider = ProviderFactory
+ .getClientProvider(comp.getArtifact());
+ compClientProvider.validateArtifact(comp.getArtifact(), fs);
+
+ if (comp.getLaunchCommand() == null && (comp.getArtifact() == null || comp
+ .getArtifact().getType() != Artifact.TypeEnum.DOCKER)) {
+ throw new IllegalArgumentException(RestApiErrorMessages
+ .ERROR_ABSENT_LAUNCH_COMMAND);
+ }
+
+ validateServiceResource(comp.getResource(), comp);
+
+ if (comp.getNumberOfContainers() == null
+ || comp.getNumberOfContainers() < 0) {
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID
+ + ": " + comp.getNumberOfContainers(), comp.getName()));
+ }
+ compClientProvider.validateConfigFiles(comp.getConfiguration()
+ .getFiles(), fs);
+
+ MonitorUtils.getProbe(comp.getReadinessCheck());
+ }
+
+ // Check component or service name format and transform to lower case.
+ public static void validateNameFormat(String name,
+ org.apache.hadoop.conf.Configuration conf) {
+ if (StringUtils.isEmpty(name)) {
+ throw new IllegalArgumentException("Name can not be empty!");
+ }
+ // validate component name
+ if (name.contains("_")) {
+ throw new IllegalArgumentException(
+ "Invalid format: " + name
+ + ", can not use '_', as DNS hostname does not allow '_'. Use '-' Instead. ");
+ }
+ boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
+ RegistryConstants.DEFAULT_DNS_ENABLED);
+ if (dnsEnabled && name.length() > RegistryConstants.MAX_FQDN_LABEL_LENGTH) {
+ throw new IllegalArgumentException(String
+ .format("Invalid format %s, must be no more than 63 characters ",
+ name));
+ }
+ namePattern.validate(name);
+ }
+
+ @VisibleForTesting
+ public static List<Component> getComponents(SliderFileSystem
+ fs, String serviceName) throws IOException {
+ return loadService(fs, serviceName).getComponents();
+ }
+
+ public static Service loadService(SliderFileSystem fs, String
+ serviceName) throws IOException {
+ Path serviceJson = getServiceJsonPath(fs, serviceName);
+ LOG.info("Loading service definition from " + serviceJson);
+ return jsonSerDeser.load(fs.getFileSystem(), serviceJson);
+ }
+
+ public static Service loadServiceFrom(SliderFileSystem fs,
+ Path appDefPath) throws IOException {
+ LOG.info("Loading service definition from " + appDefPath);
+ return jsonSerDeser.load(fs.getFileSystem(), appDefPath);
+ }
+
+ public static Path getServiceJsonPath(SliderFileSystem fs, String serviceName) {
+ Path serviceDir = fs.buildClusterDirPath(serviceName);
+ return new Path(serviceDir, serviceName + ".json");
+ }
+
+ private static void validateServiceResource(Resource resource,
+ Component comp) {
+ // Only services/components of type SERVICE can skip resource requirement
+ if (resource == null) {
+ throw new IllegalArgumentException(
+ comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String
+ .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ // One and only one of profile OR cpus & memory can be specified. Specifying
+ // both raises validation error.
+ if (StringUtils.isNotEmpty(resource.getProfile()) && (
+ resource.getCpus() != null || StringUtils
+ .isNotEmpty(resource.getMemory()))) {
+ throw new IllegalArgumentException(comp == null ?
+ RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED :
+ String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
+ comp.getName()));
+ }
+ // Currently resource profile is not supported yet, so we will raise
+ // validation error if only resource profile is specified
+ if (StringUtils.isNotEmpty(resource.getProfile())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET);
+ }
+
+ String memory = resource.getMemory();
+ Integer cpus = resource.getCpus();
+ if (StringUtils.isEmpty(memory)) {
+ throw new IllegalArgumentException(
+ comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID :
+ String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ if (cpus == null) {
+ throw new IllegalArgumentException(
+ comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID :
+ String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ if (cpus <= 0) {
+ throw new IllegalArgumentException(comp == null ?
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String
+ .format(
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
+ comp.getName()));
+ }
+ }
+
+ // check if comp mem size exceeds cluster limit
+ public static void validateCompResourceSize(
+ org.apache.hadoop.yarn.api.records.Resource maxResource,
+ Service service) throws YarnException {
+ for (Component component : service.getComponents()) {
+ // only handle mem now.
+ long mem = Long.parseLong(component.getResource().getMemory());
+ if (mem > maxResource.getMemorySize()) {
+ throw new YarnException(
+ "Component " + component.getName() + " memory size (" + mem
+ + ") is larger than configured max container memory size ("
+ + maxResource.getMemorySize() + ")");
+ }
+ }
+ }
+
+ public static boolean hasComponent(Service service) {
+ if (service.getComponents() == null || service.getComponents()
+ .isEmpty()) {
+ return false;
+ }
+ return true;
+ }
+
+ public static Component createDefaultComponent(Service service) {
+ Component comp = new Component();
+ comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME);
+ comp.setArtifact(service.getArtifact());
+ comp.setResource(service.getResource());
+ comp.setNumberOfContainers(service.getNumberOfContainers());
+ comp.setLaunchCommand(service.getLaunchCommand());
+ comp.setConfiguration(service.getConfiguration());
+ return comp;
+ }
+
+ public static Collection<Component> sortByDependencies(List<Component>
+ components) {
+ Map<String, Component> sortedComponents =
+ sortByDependencies(components, null);
+ return sortedComponents.values();
+ }
+
+ /**
+ * Each internal call of sortByDependencies will identify all of the
+ * components with the same dependency depth (the lowest depth that has not
+ * been processed yet) and add them to the sortedComponents list, preserving
+ * their original ordering in the components list.
+ *
+ * So the first time it is called, all components with no dependencies
+ * (depth 0) will be identified. The next time it is called, all components
+ * that have dependencies only on the the depth 0 components will be
+ * identified (depth 1). This will be repeated until all components have
+ * been added to the sortedComponents list. If no new components are
+ * identified but the sortedComponents list is not complete, an error is
+ * thrown.
+ */
+ private static Map<String, Component> sortByDependencies(List<Component>
+ components, Map<String, Component> sortedComponents) {
+ if (sortedComponents == null) {
+ sortedComponents = new LinkedHashMap<>();
+ }
+
+ Map<String, Component> componentsToAdd = new LinkedHashMap<>();
+ List<Component> componentsSkipped = new ArrayList<>();
+ for (Component component : components) {
+ String name = component.getName();
+ if (sortedComponents.containsKey(name)) {
+ continue;
+ }
+ boolean dependenciesAlreadySorted = true;
+ if (!SliderUtils.isEmpty(component.getDependencies())) {
+ for (String dependency : component.getDependencies()) {
+ if (!sortedComponents.containsKey(dependency)) {
+ dependenciesAlreadySorted = false;
+ break;
+ }
+ }
+ }
+ if (dependenciesAlreadySorted) {
+ componentsToAdd.put(name, component);
+ } else {
+ componentsSkipped.add(component);
+ }
+ }
+
+ if (componentsToAdd.size() == 0) {
+ throw new IllegalArgumentException(String.format(RestApiErrorMessages
+ .ERROR_DEPENDENCY_CYCLE, componentsSkipped));
+ }
+ sortedComponents.putAll(componentsToAdd);
+ if (sortedComponents.size() == components.size()) {
+ return sortedComponents;
+ }
+ return sortByDependencies(components, sortedComponents);
+ }
+
+ public static String $(String s) {
+ return "${" + s +"}";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
new file mode 100644
index 0000000..7440b11
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+
+
+public class ServiceRegistryUtils {
+
+ /**
+ * Base path for services
+ */
+ public static final String ZK_SERVICES = "services";
+
+ /**
+ * Base path for all Slider references
+ */
+ public static final String ZK_SLIDER = "slider";
+ public static final String ZK_USERS = "users";
+ public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
+ public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
+
+ /**
+ * Get the registry path for an instance under the user's home node
+ * @param instanceName application instance
+ * @return a path to the registry location for this application instance.
+ */
+ public static String registryPathForInstance(String instanceName) {
+ return RegistryUtils.servicePath(
+ RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, instanceName
+ );
+ }
+
+ /**
+ * Build the path to a cluster; exists once the cluster has come up.
+ * Even before that, a ZK watcher could wait for it.
+ * @param username user
+ * @param clustername name of the cluster
+ * @return a strin
+ */
+ public static String mkClusterPath(String username, String clustername) {
+ return mkSliderUserPath(username) + "/" + clustername;
+ }
+
+ /**
+ * Build the path to a cluster; exists once the cluster has come up.
+ * Even before that, a ZK watcher could wait for it.
+ * @param username user
+ * @return a string
+ */
+ public static String mkSliderUserPath(String username) {
+ return SVC_SLIDER_USERS + "/" + username;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
new file mode 100644
index 0000000..d6d664e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+/**
+ * Extends Core Filesystem with operations to manipulate ClusterDescription
+ * persistent state
+ */
+public class SliderFileSystem extends CoreFileSystem {
+
+ Path appDir = null;
+
+ public SliderFileSystem(FileSystem fileSystem,
+ Configuration configuration) {
+ super(fileSystem, configuration);
+ }
+
+ public SliderFileSystem(Configuration configuration) throws IOException {
+ super(configuration);
+ }
+
+ public void setAppDir(Path appDir) {
+ this.appDir = appDir;
+ }
+
+ public Path getAppDir() {
+ return this.appDir;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[57/58] [abbrv] hadoop git commit: YARN-7423. Improve service client
loggings. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7423. Improve service client loggings. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a127f7b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a127f7b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a127f7b0
Branch: refs/heads/yarn-native-services
Commit: a127f7b0fbde3942ca4329d5463bb8fe421daf7b
Parents: a6c4bd7
Author: Billie Rinaldi <bi...@apache.org>
Authored: Sat Nov 4 12:27:42 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:20 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceMaster.java | 4 +-
.../yarn/service/api/records/Configuration.java | 8 +-
.../yarn/service/client/ServiceClient.java | 50 +-
.../yarn/service/component/Component.java | 10 +-
.../component/instance/ComponentInstance.java | 4 +-
.../containerlaunch/AbstractLauncher.java | 6 +-
.../containerlaunch/ClasspathConstructor.java | 4 +-
.../containerlaunch/CommandLineBuilder.java | 4 +-
.../containerlaunch/JavaCommandLineBuilder.java | 4 +-
.../yarn/service/monitor/probe/HttpProbe.java | 4 +-
.../yarn/service/monitor/probe/PortProbe.java | 4 +-
.../provider/AbstractClientProvider.java | 8 +-
.../provider/AbstractProviderService.java | 4 +-
.../yarn/service/provider/ProviderUtils.java | 10 +-
.../registry/YarnRegistryViewForProviders.java | 6 +-
.../yarn/service/utils/CoreFileSystem.java | 2 +-
.../hadoop/yarn/service/utils/PortScanner.java | 4 +-
.../yarn/service/utils/ServiceApiUtil.java | 2 +-
.../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++++++++++++++++++
.../hadoop/yarn/service/utils/SliderUtils.java | 564 -------------------
.../yarn/service/utils/ZookeeperUtils.java | 4 +-
.../service/conf/TestValidateServiceNames.java | 1 -
22 files changed, 614 insertions(+), 638 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index 2d86a5a..b0b4f06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,7 +112,7 @@ public class ServiceMaster extends CompositeService {
}
protected ContainerId getAMContainerId() throws BadClusterStateException {
- return ContainerId.fromString(SliderUtils.mandatoryEnvVariable(
+ return ContainerId.fromString(ServiceUtils.mandatoryEnvVariable(
ApplicationConstants.Environment.CONTAINER_ID.name()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
index dc6dff8..859c1ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
@@ -24,7 +24,7 @@ import io.swagger.annotations.ApiModelProperty;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.io.Serializable;
import java.util.ArrayList;
@@ -201,9 +201,9 @@ public class Configuration implements Serializable {
* this ConfigFile.
*/
public synchronized void mergeFrom(Configuration that) {
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
+ ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
.getProperties());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
+ ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
Map<String, ConfigFile> thatMap = new HashMap<>();
for (ConfigFile file : that.getFiles()) {
@@ -212,7 +212,7 @@ public class Configuration implements Serializable {
for (ConfigFile thisFile : files) {
if(thatMap.containsKey(thisFile.getDestFile())) {
ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
+ ServiceUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
thatFile.getProperties());
thatMap.remove(thisFile.getDestFile());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 775a642..11cd30d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -69,7 +69,7 @@ import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.Times;
@@ -80,19 +80,13 @@ import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.text.MessageFormat;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
-import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
+import static org.apache.hadoop.yarn.service.utils.ServiceUtils.*;
@InterfaceAudience.Public
@InterfaceStability.Unstable
@@ -169,7 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
fileName);
}
Path filePath = new Path(file.getAbsolutePath());
- LOG.info("Loading service definition from: " + filePath);
+ LOG.info("Loading service definition from local FS: " + filePath);
Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
if (!StringUtils.isEmpty(serviceName)) {
@@ -482,7 +476,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
// though if neither is set: trouble
- if (SliderUtils.isUnset(registryQuorum)) {
+ if (ServiceUtils.isUnset(registryQuorum)) {
throw new BadConfigException(
"No Zookeeper quorum provided in the" + " configuration property "
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
@@ -505,7 +499,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
types.add(YarnServiceConstants.APP_TYPE);
Set<String> tags = null;
if (serviceName != null) {
- tags = Collections.singleton(SliderUtils.createNameTag(serviceName));
+ tags = Collections.singleton(ServiceUtils.createNameTag(serviceName));
}
GetApplicationsRequest request = GetApplicationsRequest.newInstance();
request.setApplicationTypes(types);
@@ -652,7 +646,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
// write out the path output
CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
String cmdStr = CLI.build();
- LOG.info("AM launch command: {}", cmdStr);
+ LOG.debug("AM launch command: {}", cmdStr);
return cmdStr;
}
@@ -671,12 +665,12 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
}
if (!UserGroupInformation.isSecurityEnabled()) {
String userName = UserGroupInformation.getCurrentUser().getUserName();
- LOG.info("Run as user " + userName);
+ LOG.debug("Run as user " + userName);
// HADOOP_USER_NAME env is used by UserGroupInformation when log in
// This env makes AM run as this user
env.put("HADOOP_USER_NAME", userName);
}
- LOG.info("AM env: \n{}", stringifyMap(env));
+ LOG.debug("AM env: \n{}", stringifyMap(env));
return env;
}
@@ -689,11 +683,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
libPath, "lib", false);
Path dependencyLibTarGzip = fs.getDependencyTarGzip();
if (fs.isFile(dependencyLibTarGzip)) {
- LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ dependencyLibTarGzip);
- SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
+ fs.submitTarGzipAndUpdate(localResources);
} else {
- String[] libs = SliderUtils.getLibDirs();
+ String[] libs = ServiceUtils.getLibDirs();
+ LOG.info("Uploading all dependency jars to HDFS. For faster submission of" +
+ " apps, pre-upload dependency jars to HDFS "
+ + "using command: yarn app -enableFastLaunch");
for (String libDirProp : libs) {
ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
libDirProp);
@@ -740,7 +737,8 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
ApplicationId appId = submitApp(service);
service.setId(appId.toString());
// write app definition on to hdfs
- createDirAndPersistApp(appDir, service);
+ Path appJson = persistAppDef(appDir, service);
+ LOG.info("Persisted service " + service.getName() + " at " + appJson);
return 0;
}
@@ -763,16 +761,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
throws IOException, SliderException {
FsPermission appDirPermission = new FsPermission("750");
fs.createWithPermissions(appDir, appDirPermission);
- persistAppDef(appDir, service);
+ Path appJson = persistAppDef(appDir, service);
+ LOG.info("Persisted service " + service.getName() + " at " + appJson);
}
- private void persistAppDef(Path appDir, Service service)
- throws IOException {
+ private Path persistAppDef(Path appDir, Service service) throws IOException {
Path appJson = new Path(appDir, service.getName() + ".json");
- jsonSerDeser
- .save(fs.getFileSystem(), appJson, service, true);
- LOG.info(
- "Persisted service " + service.getName() + " at " + appJson);
+ jsonSerDeser.save(fs.getFileSystem(), appJson, service, true);
+ return appJson;
}
private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
@@ -922,7 +918,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return EXIT_SUCCESS;
}
- String[] libDirs = SliderUtils.getLibDirs();
+ String[] libDirs = ServiceUtils.getLibDirs();
if (libDirs.length > 0) {
File tempLibTarGzipFile = File.createTempFile(
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 7667a53..7208f39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
import org.apache.hadoop.yarn.service.monitor.probe.Probe;
import org.slf4j.Logger;
@@ -419,7 +419,7 @@ public class Component implements EventHandler<ComponentEvent> {
public boolean areDependenciesReady() {
List<String> dependencies = componentSpec.getDependencies();
- if (SliderUtils.isEmpty(dependencies)) {
+ if (ServiceUtils.isEmpty(dependencies)) {
return true;
}
for (String dependency : dependencies) {
@@ -445,7 +445,7 @@ public class Component implements EventHandler<ComponentEvent> {
public Map<String, String> getDependencyHostIpTokens() {
Map<String, String> tokens = new HashMap<>();
List<String> dependencies = componentSpec.getDependencies();
- if (SliderUtils.isEmpty(dependencies)) {
+ if (ServiceUtils.isEmpty(dependencies)) {
return tokens;
}
for (String dependency : dependencies) {
@@ -455,8 +455,8 @@ public class Component implements EventHandler<ComponentEvent> {
if (instance.getContainerStatus() == null) {
continue;
}
- if (SliderUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
- SliderUtils.isUnset(instance.getContainerStatus().getHost())) {
+ if (ServiceUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
+ ServiceUtils.isUnset(instance.getContainerStatus().getHost())) {
continue;
}
String ip = instance.getContainerStatus().getIPs().get(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 2492454..9e5f98c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
@@ -503,7 +503,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
+ nodeId + ", will try again", e);
return;
}
- if (SliderUtils.isEmpty(status.getIPs()) || SliderUtils
+ if (ServiceUtils.isEmpty(status.getIPs()) || ServiceUtils
.isUnset(status.getHost())) {
return;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
index e4eae20..2d7c3bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -132,7 +132,7 @@ public class AbstractLauncher {
*/
public ContainerLaunchContext completeContainerLaunch() throws IOException {
- String cmdStr = SliderUtils.join(commands, " ", false);
+ String cmdStr = ServiceUtils.join(commands, " ", false);
log.debug("Completed setting up container command {}", cmdStr);
containerLaunchContext.setCommands(commands);
@@ -205,7 +205,7 @@ public class AbstractLauncher {
String key = entry.getKey();
LocalResource val = entry.getValue();
- log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
+ log.debug(key + "=" + ServiceUtils.stringify(val.getResource()));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
index 22b3877..711abb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.util.ArrayList;
import java.util.Arrays;
@@ -63,7 +63,7 @@ public class ClasspathConstructor {
}
public String buildClasspath() {
- return SliderUtils.join(pathElements,
+ return ServiceUtils.join(pathElements,
CLASS_PATH_SEPARATOR,
false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
index 7baa284..5ed56e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.util.ArrayList;
import java.util.List;
@@ -81,6 +81,6 @@ public class CommandLineBuilder {
* @return the command line
*/
public String build() {
- return SliderUtils.join(argumentList, " ");
+ return ServiceUtils.join(argumentList, " ");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
index cbcb0d6..7f6cc04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
@@ -22,7 +22,7 @@ package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
import java.util.Map;
@@ -53,7 +53,7 @@ public class JavaCommandLineBuilder extends CommandLineBuilder {
* trimmed.
*/
public void setJVMHeap(String heap) {
- if (SliderUtils.isSet(heap)) {
+ if (ServiceUtils.isSet(heap)) {
add("-Xmx" + heap.trim());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
index 1923086..1ed13a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
@@ -21,7 +21,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,7 +75,7 @@ public class HttpProbe extends Probe {
public ProbeStatus ping(ComponentInstance instance) {
ProbeStatus status = new ProbeStatus();
ContainerStatus containerStatus = instance.getContainerStatus();
- if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
+ if (containerStatus == null || ServiceUtils.isEmpty(containerStatus.getIPs())
|| StringUtils.isEmpty(containerStatus.getHost())) {
status.fail(this, new IOException("IP is not available yet"));
return status;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
index aba5859..85569f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.service.monitor.probe;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,7 +67,7 @@ public class PortProbe extends Probe {
public ProbeStatus ping(ComponentInstance instance) {
ProbeStatus status = new ProbeStatus();
- if (instance.getContainerStatus() == null || SliderUtils
+ if (instance.getContainerStatus() == null || ServiceUtils
.isEmpty(instance.getContainerStatus().getIPs())) {
status.fail(this, new IOException(
instance.getCompInstanceName() + ": IP is not available yet"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
index 75c647b..fc8953c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.io.IOException;
import java.nio.file.Paths;
@@ -53,12 +53,12 @@ public abstract class AbstractClientProvider {
public static final Set<String> createApplicationTags(String appName,
String appVersion, String appDescription) {
Set<String> tags = new HashSet<>();
- tags.add(SliderUtils.createNameTag(appName));
+ tags.add(ServiceUtils.createNameTag(appName));
if (appVersion != null) {
- tags.add(SliderUtils.createVersionTag(appVersion));
+ tags.add(ServiceUtils.createVersionTag(appVersion));
}
if (appDescription != null) {
- tags.add(SliderUtils.createDescriptionTag(appDescription));
+ tags.add(ServiceUtils.createDescriptionTag(appDescription));
}
return tags;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 6ffb84d..6d74061 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
@@ -70,7 +70,7 @@ public abstract class AbstractProviderService implements ProviderService,
.initCompTokensForSubstitute(instance);
tokensForSubstitution.putAll(globalTokens);
// Set the environment variables in launcher
- launcher.putEnv(SliderUtils
+ launcher.putEnv(ServiceUtils
.buildEnvMap(component.getConfiguration(), tokensForSubstitution));
launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 582c5ac..e074dd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -94,7 +94,7 @@ public class ProviderUtils implements YarnServiceConstants {
IOException,
SliderException {
try {
- SliderUtils.putJar(providerResources,
+ ServiceUtils.putJar(providerResources,
sliderFileSystem,
providerClass,
tempPath,
@@ -127,14 +127,14 @@ public class ProviderUtils implements YarnServiceConstants {
String libDir,
String libLocalSrcDir)
throws IOException, SliderException {
- if (SliderUtils.isSet(libLocalSrcDir)) {
+ if (ServiceUtils.isSet(libLocalSrcDir)) {
File file = new File(libLocalSrcDir);
if (!file.exists() || !file.isDirectory()) {
throw new BadCommandArgumentsException(
"Supplied lib src dir %s is not valid", libLocalSrcDir);
}
}
- SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
+ ServiceUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
libDir, libLocalSrcDir);
}
@@ -174,7 +174,7 @@ public class ProviderUtils implements YarnServiceConstants {
Configuration conf = service.getConfiguration();
String keytabPathOnHost =
conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
- if (SliderUtils.isUnset(keytabPathOnHost)) {
+ if (ServiceUtils.isUnset(keytabPathOnHost)) {
String amKeytabName =
conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
String keytabDir =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
index d418b59..cecca5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,9 +70,9 @@ public class YarnRegistryViewForProviders {
Preconditions.checkArgument(registryOperations != null,
"null registry operations");
Preconditions.checkArgument(user != null, "null user");
- Preconditions.checkArgument(SliderUtils.isSet(serviceClass),
+ Preconditions.checkArgument(ServiceUtils.isSet(serviceClass),
"unset service class");
- Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+ Preconditions.checkArgument(ServiceUtils.isSet(instanceName),
"instanceName");
Preconditions.checkArgument(applicationAttemptId != null,
"null applicationAttemptId");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
index 281e1dfe..6b9d4d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -409,7 +409,7 @@ public class CoreFileSystem {
*/
public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
throws IOException, SliderException {
- File localFile = SliderUtils.findContainingJarOrFail(clazz);
+ File localFile = ServiceUtils.findContainingJarOrFail(clazz);
return submitFile(localFile, tempPath, subdir, jarName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
index 2dbf37f..1d64ed6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
@@ -85,7 +85,7 @@ public class PortScanner {
if (remainingPortsToCheck != null) {
return getAvailablePortViaPortArray();
} else {
- return SliderUtils.getOpenPort();
+ return ServiceUtils.getOpenPort();
}
}
@@ -95,7 +95,7 @@ public class PortScanner {
Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
while (portsToCheck.hasNext() && !found) {
int portToCheck = portsToCheck.next();
- found = SliderUtils.isPortAvailable(portToCheck);
+ found = ServiceUtils.isPortAvailable(portToCheck);
if (found) {
availablePort = portToCheck;
portsToCheck.remove();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 68db0bb..b58cea8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -369,7 +369,7 @@ public class ServiceApiUtil {
continue;
}
boolean dependenciesAlreadySorted = true;
- if (!SliderUtils.isEmpty(component.getDependencies())) {
+ if (!ServiceUtils.isEmpty(component.getDependencies())) {
for (String dependency : component.getDependencies()) {
if (!sortedComponents.containsKey(dependency)) {
dependenciesAlreadySorted = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
new file mode 100644
index 0000000..e18bcae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class ServiceUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(ServiceUtils.class);
+
+ private ServiceUtils() {
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s string
+ * @return true iff the string is neither null nor empty
+ */
+ public static boolean isUnset(String s) {
+ return s == null || s.isEmpty();
+ }
+
+ public static boolean isSet(String s) {
+ return !isUnset(s);
+ }
+
+ public static boolean isEmpty(Collection l) {
+ return l == null || l.isEmpty();
+ }
+
+ /**
+ * Find a containing JAR
+ * @param clazz class to find
+ * @return the file
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ * @throws FileNotFoundException if the class did not resolve to a file
+ */
+ public static File findContainingJarOrFail(Class clazz) throws IOException {
+ File localFile = ServiceUtils.findContainingJar(clazz);
+ if (null == localFile) {
+ throw new FileNotFoundException("Could not find JAR containing " + clazz);
+ }
+ return localFile;
+ }
+
+
+ /**
+ * Find a containing JAR
+ * @param my_class class to find
+ * @return the file or null if it is not found
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ */
+ public static File findContainingJar(Class my_class) throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ if (loader == null) {
+ throw new IOException(
+ "Class " + my_class + " does not have a classloader!");
+ }
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+ Enumeration<URL> urlEnumeration = loader.getResources(class_file);
+ for (; urlEnumeration.hasMoreElements(); ) {
+ URL url = urlEnumeration.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ String jarFilePath = toReturn.replaceAll("!.*$", "");
+ return new File(jarFilePath);
+ } else {
+ log.info("could not locate JAR containing {} URL={}", my_class, url);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Copy a file to a new FS -both paths must be qualified.
+ * @param conf conf file
+ * @param srcFile src file
+ * @param destFile dest file
+ */
+ @SuppressWarnings("deprecation")
+ public static void copy(Configuration conf,
+ Path srcFile,
+ Path destFile) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcFile)) {
+ throw new FileNotFoundException("Source file not found " + srcFile);
+ }
+ if (!srcFS.isFile(srcFile)) {
+ throw new FileNotFoundException(
+ "Source file not a file " + srcFile);
+ }
+ FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
+ FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
+ }
+
+ /**
+ * Take a collection, return a list containing the string value of every
+ * element in the collection.
+ * @param c collection
+ * @return a stringified list
+ */
+ public static List<String> collectionToStringList(Collection c) {
+ List<String> l = new ArrayList<>(c.size());
+ for (Object o : c) {
+ l.add(o.toString());
+ }
+ return l;
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(Collection collection, String separator) {
+ return join(collection, separator, true);
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
+ StringBuilder b = new StringBuilder();
+ // fast return on empty collection
+ if (collection.isEmpty()) {
+ return trailing ? separator : "";
+ }
+ for (Object o : collection) {
+ b.append(o);
+ b.append(separator);
+ }
+ int length = separator.length();
+ String s = b.toString();
+ return (trailing || s.isEmpty()) ?
+ s : (b.substring(0, b.length() - length));
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection strings
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator,
+ boolean trailing) {
+ return join(Arrays.asList(collection), separator, trailing);
+ }
+
+ /**
+ * Resolve a mandatory environment variable
+ * @param key env var
+ * @return the resolved value
+ * @throws BadClusterStateException
+ */
+ public static String mandatoryEnvVariable(String key) throws
+ BadClusterStateException {
+ String v = System.getenv(key);
+ if (v == null) {
+ throw new BadClusterStateException("Missing Environment variable " + key);
+ }
+ return v;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ Preconditions.checkArgument(first != null, "Null 'first' value");
+ Preconditions.checkArgument(second != null, "Null 'second' value");
+ for (Map.Entry<T1, T2> entry : second.entrySet()) {
+ T1 key = entry.getKey();
+ if (!first.containsKey(key)) {
+ first.put(key, entry.getValue());
+ }
+ }
+ return first;
+ }
+
+ /**
+ * Convert a map to a multi-line string for printing
+ * @param map map to stringify
+ * @return a string representation of the map
+ */
+ public static String stringifyMap(Map<String, String> map) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ builder.append(entry.getKey())
+ .append("=\"")
+ .append(entry.getValue())
+ .append("\"\n");
+
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Convert a YARN URL into a string value of a normal URL
+ * @param url URL
+ * @return string representatin
+ */
+ public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(url.getScheme()).append("://");
+ if (url.getHost() != null) {
+ builder.append(url.getHost()).append(":").append(url.getPort());
+ }
+ builder.append(url.getFile());
+ return builder.toString();
+ }
+
+ /**
+ * Get a random open port
+ * @return true if the port was available for listening on
+ */
+ public static int getOpenPort() throws IOException {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ return socket.getLocalPort();
+ } finally {
+ if (socket != null) {
+ socket.close();
+ }
+ }
+ }
+
+ /**
+ * See if a port is available for listening on by trying to listen
+ * on it and seeing if that works or fails.
+ * @param port port to listen to
+ * @return true if the port was available for listening on
+ */
+ public static boolean isPortAvailable(int port) {
+ try {
+ ServerSocket socket = new ServerSocket(port);
+ socket.close();
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ // Build env map: key -> value;
+ // value will be replaced by the corresponding value in tokenMap, if any.
+ public static Map<String, String> buildEnvMap(
+ org.apache.hadoop.yarn.service.api.records.Configuration conf,
+ Map<String,String> tokenMap) {
+ if (tokenMap == null) {
+ return conf.getEnv();
+ }
+ Map<String, String> env = new HashMap<>();
+ for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()),
+ token.getValue());
+ }
+ env.put(key,val);
+ }
+ return env;
+ }
+
+ public static String[] getLibDirs() {
+ String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
+ if (isUnset(libDirStr)) {
+ return ArrayUtils.EMPTY_STRING_ARRAY;
+ }
+ return StringUtils.split(libDirStr, ',');
+ }
+
+ /**
+ * Submit a JAR containing a specific class and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param clazz class to look for
+ * @param libdir lib directory
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public static LocalResource putJar(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
+ LocalResource res = sliderFileSystem.submitJarWithClass(
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
+ return res;
+ }
+
+ /**
+ * Submit a JAR containing and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param libDir lib directory
+ * @param srcPath copy jars from
+ */
+ public static void putAllJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String srcPath) throws IOException, SliderException {
+ log.debug("Loading all dependencies from {}", srcPath);
+ if (ServiceUtils.isSet(srcPath)) {
+ File srcFolder = new File(srcPath);
+ FilenameFilter jarFilter = createJarFilter();
+ File[] listOfJars = srcFolder.listFiles(jarFilter);
+ if (listOfJars == null || listOfJars.length == 0) {
+ return;
+ }
+ for (File jarFile : listOfJars) {
+ LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+ providerResources.put(libDir + "/" + jarFile.getName(), res);
+ }
+ }
+ }
+
+ /**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
+ * Create a file:// path from a local file
+ * @param file file to point the path
+ * @return a new Path
+ */
+ public static Path createLocalPath(File file) {
+ return new Path(file.toURI());
+ }
+
+ /**
+ * Build up the classpath for execution
+ * -behaves very differently on a mini test cluster vs a production
+ * production one.
+ *
+ * @param sliderConfDir relative path to the dir containing slider config
+ * options to put on the classpath -or null
+ * @param libdir directory containing the JAR files
+ * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
+ * (and hence the current classpath should be used, not anything built up)
+ * @return a classpath
+ */
+ public static ClasspathConstructor buildClasspath(String sliderConfDir,
+ String libdir,
+ SliderFileSystem sliderFileSystem,
+ boolean usingMiniMRCluster) {
+
+ ClasspathConstructor classpath = new ClasspathConstructor();
+ classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+
+ // add the runtime classpath needed for tests to work
+ if (usingMiniMRCluster) {
+ // for mini cluster we pass down the java CP properties
+ // and nothing else
+ classpath.appendAll(classpath.localJVMClasspath());
+ } else {
+ if (sliderConfDir != null) {
+ classpath.addClassDirectory(sliderConfDir);
+ }
+ classpath.addLibDir(libdir);
+ if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
+ classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
+ }
+ classpath.addRemoteClasspathEnvVar();
+ classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
+ }
+ return classpath;
+ }
+
+ /**
+ * Given a source folder create a tar.gz file
+ *
+ * @param libDirs
+ * @param tarGzipFile
+ *
+ * @throws IOException
+ */
+ public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
+ FilenameFilter filter) throws IOException {
+ log.info("Tar-gzipping folders {} to {}", libDirs,
+ tarGzipFile.getAbsolutePath());
+
+ try(TarArchiveOutputStream taos =
+ new TarArchiveOutputStream(new GZIPOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
+ for (String libDir : libDirs) {
+ File srcFolder = new File(libDir);
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true, filter);
+ for (String file : files) {
+ File srcFile = new File(srcFolder, file);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ srcFile, file);
+ taos.putArchiveEntry(tarEntry);
+ try(FileInputStream in = new FileInputStream(srcFile)) {
+ org.apache.commons.io.IOUtils.copy(in, taos);
+ }
+ taos.flush();
+ taos.closeArchiveEntry();
+ }
+ }
+ }
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative, FilenameFilter filter) {
+ if (node.isFile()) {
+ String fileFullPath = node.toString();
+ if (relative) {
+ fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
+ fileFullPath.length()));
+ } else {
+ fileList.add(fileFullPath);
+ }
+ }
+
+ if (node.isDirectory()) {
+ String[] subNode = node.list(filter);
+ if (subNode == null || subNode.length == 0) {
+ return;
+ }
+ for (String filename : subNode) {
+ generateFileList(fileList, new File(node, filename), rootFolder,
+ relative, filter);
+ }
+ }
+ }
+
+ public static String createNameTag(String name) {
+ return "Name: " + name;
+ }
+
+ public static String createVersionTag(String version) {
+ return "Version: " + version;
+ }
+
+ public static String createDescriptionTag(String description) {
+ return "Description: " + description;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
deleted file mode 100644
index 2809dfb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ /dev/null
@@ -1,564 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPOutputStream;
-
-/**
- * These are slider-specific Util methods
- */
-public final class SliderUtils {
-
- private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
-
- private SliderUtils() {
- }
-
- /**
- * Implementation of set-ness, groovy definition of true/false for a string
- * @param s string
- * @return true iff the string is neither null nor empty
- */
- public static boolean isUnset(String s) {
- return s == null || s.isEmpty();
- }
-
- public static boolean isSet(String s) {
- return !isUnset(s);
- }
-
- public static boolean isEmpty(Collection l) {
- return l == null || l.isEmpty();
- }
-
- /**
- * Find a containing JAR
- * @param clazz class to find
- * @return the file
- * @throws IOException any IO problem, including the class not having a
- * classloader
- * @throws FileNotFoundException if the class did not resolve to a file
- */
- public static File findContainingJarOrFail(Class clazz) throws IOException {
- File localFile = SliderUtils.findContainingJar(clazz);
- if (null == localFile) {
- throw new FileNotFoundException("Could not find JAR containing " + clazz);
- }
- return localFile;
- }
-
-
- /**
- * Find a containing JAR
- * @param my_class class to find
- * @return the file or null if it is not found
- * @throws IOException any IO problem, including the class not having a
- * classloader
- */
- public static File findContainingJar(Class my_class) throws IOException {
- ClassLoader loader = my_class.getClassLoader();
- if (loader == null) {
- throw new IOException(
- "Class " + my_class + " does not have a classloader!");
- }
- String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
- Enumeration<URL> urlEnumeration = loader.getResources(class_file);
- for (; urlEnumeration.hasMoreElements(); ) {
- URL url = urlEnumeration.nextElement();
- if ("jar".equals(url.getProtocol())) {
- String toReturn = url.getPath();
- if (toReturn.startsWith("file:")) {
- toReturn = toReturn.substring("file:".length());
- }
- // URLDecoder is a misnamed class, since it actually decodes
- // x-www-form-urlencoded MIME type rather than actual
- // URL encoding (which the file path has). Therefore it would
- // decode +s to ' 's which is incorrect (spaces are actually
- // either unencoded or encoded as "%20"). Replace +s first, so
- // that they are kept sacred during the decoding process.
- toReturn = toReturn.replaceAll("\\+", "%2B");
- toReturn = URLDecoder.decode(toReturn, "UTF-8");
- String jarFilePath = toReturn.replaceAll("!.*$", "");
- return new File(jarFilePath);
- } else {
- log.info("could not locate JAR containing {} URL={}", my_class, url);
- }
- }
- return null;
- }
-
- /**
- * Copy a file to a new FS -both paths must be qualified.
- * @param conf conf file
- * @param srcFile src file
- * @param destFile dest file
- */
- @SuppressWarnings("deprecation")
- public static void copy(Configuration conf,
- Path srcFile,
- Path destFile) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcFile)) {
- throw new FileNotFoundException("Source file not found " + srcFile);
- }
- if (!srcFS.isFile(srcFile)) {
- throw new FileNotFoundException(
- "Source file not a file " + srcFile);
- }
- FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
- FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
- }
-
- /**
- * Take a collection, return a list containing the string value of every
- * element in the collection.
- * @param c collection
- * @return a stringified list
- */
- public static List<String> collectionToStringList(Collection c) {
- List<String> l = new ArrayList<>(c.size());
- for (Object o : c) {
- l.add(o.toString());
- }
- return l;
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -including at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(Collection collection, String separator) {
- return join(collection, separator, true);
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(Collection collection,
- String separator,
- boolean trailing) {
- StringBuilder b = new StringBuilder();
- // fast return on empty collection
- if (collection.isEmpty()) {
- return trailing ? separator : "";
- }
- for (Object o : collection) {
- b.append(o);
- b.append(separator);
- }
- int length = separator.length();
- String s = b.toString();
- return (trailing || s.isEmpty()) ?
- s : (b.substring(0, b.length() - length));
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection strings
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(String[] collection, String separator,
- boolean trailing) {
- return join(Arrays.asList(collection), separator, trailing);
- }
-
- /**
- * Resolve a mandatory environment variable
- * @param key env var
- * @return the resolved value
- * @throws BadClusterStateException
- */
- public static String mandatoryEnvVariable(String key) throws
- BadClusterStateException {
- String v = System.getenv(key);
- if (v == null) {
- throw new BadClusterStateException("Missing Environment variable " + key);
- }
- return v;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
- Map<T1, T2> second) {
- Preconditions.checkArgument(first != null, "Null 'first' value");
- Preconditions.checkArgument(second != null, "Null 'second' value");
- for (Map.Entry<T1, T2> entry : second.entrySet()) {
- T1 key = entry.getKey();
- if (!first.containsKey(key)) {
- first.put(key, entry.getValue());
- }
- }
- return first;
- }
-
- /**
- * Convert a map to a multi-line string for printing
- * @param map map to stringify
- * @return a string representation of the map
- */
- public static String stringifyMap(Map<String, String> map) {
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, String> entry : map.entrySet()) {
- builder.append(entry.getKey())
- .append("=\"")
- .append(entry.getValue())
- .append("\"\n");
-
- }
- return builder.toString();
- }
-
- /**
- * Convert a YARN URL into a string value of a normal URL
- * @param url URL
- * @return string representatin
- */
- public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
- StringBuilder builder = new StringBuilder();
- builder.append(url.getScheme()).append("://");
- if (url.getHost() != null) {
- builder.append(url.getHost()).append(":").append(url.getPort());
- }
- builder.append(url.getFile());
- return builder.toString();
- }
-
- /**
- * Get a random open port
- * @return true if the port was available for listening on
- */
- public static int getOpenPort() throws IOException {
- ServerSocket socket = null;
- try {
- socket = new ServerSocket(0);
- return socket.getLocalPort();
- } finally {
- if (socket != null) {
- socket.close();
- }
- }
- }
-
- /**
- * See if a port is available for listening on by trying to listen
- * on it and seeing if that works or fails.
- * @param port port to listen to
- * @return true if the port was available for listening on
- */
- public static boolean isPortAvailable(int port) {
- try {
- ServerSocket socket = new ServerSocket(port);
- socket.close();
- return true;
- } catch (IOException e) {
- return false;
- }
- }
-
- // Build env map: key -> value;
- // value will be replaced by the corresponding value in tokenMap, if any.
- public static Map<String, String> buildEnvMap(
- org.apache.hadoop.yarn.service.api.records.Configuration conf,
- Map<String,String> tokenMap) {
- if (tokenMap == null) {
- return conf.getEnv();
- }
- Map<String, String> env = new HashMap<>();
- for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- for (Map.Entry<String,String> token : tokenMap.entrySet()) {
- val = val.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- }
- env.put(key,val);
- }
- return env;
- }
-
- public static String[] getLibDirs() {
- String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
- if (isUnset(libDirStr)) {
- return ArrayUtils.EMPTY_STRING_ARRAY;
- }
- return StringUtils.split(libDirStr, ',');
- }
-
- /**
- * Submit a JAR containing a specific class and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param clazz class to look for
- * @param libdir lib directory
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public static LocalResource putJar(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Class clazz,
- Path tempPath,
- String libdir,
- String jarName
- )
- throws IOException, SliderException {
- LocalResource res = sliderFileSystem.submitJarWithClass(
- clazz,
- tempPath,
- libdir,
- jarName);
- providerResources.put(libdir + "/" + jarName, res);
- return res;
- }
-
- /**
- * Submit a JAR containing and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param libDir lib directory
- * @param srcPath copy jars from
- */
- public static void putAllJars(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String srcPath) throws IOException, SliderException {
- log.info("Loading all dependencies from {}", srcPath);
- if (SliderUtils.isSet(srcPath)) {
- File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = createJarFilter();
- File[] listOfJars = srcFolder.listFiles(jarFilter);
- if (listOfJars == null || listOfJars.length == 0) {
- return;
- }
- for (File jarFile : listOfJars) {
- LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
- providerResources.put(libDir + "/" + jarFile.getName(), res);
- }
- }
- }
-
- /**
- * Accept all filenames ending with {@code .jar}
- * @return a filename filter
- */
- public static FilenameFilter createJarFilter() {
- return new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
- }
- };
- }
-
- /**
- * Submit the AM tar.gz containing all dependencies and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- */
- public static void putAmTarGzipAndUpdate(
- Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem
- ) throws IOException, SliderException {
- log.info("Loading all dependencies from {}{}",
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
- sliderFileSystem.submitTarGzipAndUpdate(providerResources);
- }
-
- /**
- * Create a file:// path from a local file
- * @param file file to point the path
- * @return a new Path
- */
- public static Path createLocalPath(File file) {
- return new Path(file.toURI());
- }
-
- /**
- * Build up the classpath for execution
- * -behaves very differently on a mini test cluster vs a production
- * production one.
- *
- * @param sliderConfDir relative path to the dir containing slider config
- * options to put on the classpath -or null
- * @param libdir directory containing the JAR files
- * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
- * (and hence the current classpath should be used, not anything built up)
- * @return a classpath
- */
- public static ClasspathConstructor buildClasspath(String sliderConfDir,
- String libdir,
- SliderFileSystem sliderFileSystem,
- boolean usingMiniMRCluster) {
-
- ClasspathConstructor classpath = new ClasspathConstructor();
- classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
-
- // add the runtime classpath needed for tests to work
- if (usingMiniMRCluster) {
- // for mini cluster we pass down the java CP properties
- // and nothing else
- classpath.appendAll(classpath.localJVMClasspath());
- } else {
- if (sliderConfDir != null) {
- classpath.addClassDirectory(sliderConfDir);
- }
- classpath.addLibDir(libdir);
- if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
- classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
- } else {
- log.info(
- "For faster submission of apps, upload dependencies using cmd " +
- "enableFastLaunch");
- }
- classpath.addRemoteClasspathEnvVar();
- classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
- }
- return classpath;
- }
-
- /**
- * Given a source folder create a tar.gz file
- *
- * @param libDirs
- * @param tarGzipFile
- *
- * @throws IOException
- */
- public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
- FilenameFilter filter) throws IOException {
- log.info("Tar-gzipping folders {} to {}", libDirs,
- tarGzipFile.getAbsolutePath());
-
- try(TarArchiveOutputStream taos =
- new TarArchiveOutputStream(new GZIPOutputStream(
- new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
- for (String libDir : libDirs) {
- File srcFolder = new File(libDir);
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true, filter);
- for (String file : files) {
- File srcFile = new File(srcFolder, file);
- TarArchiveEntry tarEntry = new TarArchiveEntry(
- srcFile, file);
- taos.putArchiveEntry(tarEntry);
- try(FileInputStream in = new FileInputStream(srcFile)) {
- org.apache.commons.io.IOUtils.copy(in, taos);
- }
- taos.flush();
- taos.closeArchiveEntry();
- }
- }
- }
- }
-
- private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative, FilenameFilter filter) {
- if (node.isFile()) {
- String fileFullPath = node.toString();
- if (relative) {
- fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
- fileFullPath.length()));
- } else {
- fileList.add(fileFullPath);
- }
- }
-
- if (node.isDirectory()) {
- String[] subNode = node.list(filter);
- if (subNode == null || subNode.length == 0) {
- return;
- }
- for (String filename : subNode) {
- generateFileList(fileList, new File(node, filename), rootFolder,
- relative, filter);
- }
- }
- }
-
- public static String createNameTag(String name) {
- return "Name: " + name;
- }
-
- public static String createVersionTag(String version) {
- return "Version: " + version;
- }
-
- public static String createDescriptionTag(String description) {
- return "Description: " + description;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
index 1fa07ce..c6e8525 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
@@ -32,7 +32,7 @@ public class ZookeeperUtils {
String zkPort = Integer.toString(port);
//parse the hosts
String[] hostlist = zkHosts.split(",", 0);
- String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
+ String quorum = ServiceUtils.join(hostlist, ":" + zkPort + ",", false);
return quorum;
}
@@ -117,7 +117,7 @@ public class ZookeeperUtils {
for (HostAndPort hostAndPort : hostAndPorts) {
entries.add(buildQuorumEntry(hostAndPort, defaultPort));
}
- return SliderUtils.join(entries, ",", false);
+ return ServiceUtils.join(entries, ",", false);
}
public static String convertToHostsOnlyList(String quorum) throws
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
index 6159215..d7fa9a04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.service.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.junit.Assert;
import org.junit.Test;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[55/58] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN
related params in the service REST API. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
index 17d38cb..0f0156a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
@@ -20,7 +20,7 @@
<div class="row">
<div class="panel panel-default">
<div class="panel-heading">
- <div class="panel-title">{{type}} Metrics: Detailed Information</div>
+ <div class="panel-title">{{type}} Metrics:</div>
</div>
<div class="">
<table class="table table-hover table-custom-bordered table-custom-stripped table-radius-none table-border-none">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
index 5a94575..ef517d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
@@ -20,7 +20,7 @@
{{#if model.container}}
<div class="panel panel-default">
<div class="panel-heading">
- <div class="panel-title">Component Information</div>
+ <div class="panel-title">Component Instance Information</div>
</div>
<div class="">
<table class="table table-striped table-bordered table-hover">
@@ -30,7 +30,7 @@
<td>{{check-availability model.container.instanceName}}</td>
</tr>
<tr>
- <td>Component Group</td>
+ <td>Component</td>
<td>{{check-availability model.container.component}}</td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
index 0b642b0..fb4dba2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
@@ -18,7 +18,7 @@
<div class="row">
<div class="col-md-12">
- <h3>Active Components: {{model.componentName}}</h3>
+ <h3>Component: {{model.componentName}}</h3>
{{em-table columns=tableColumns rows=model.instances}}
</div>
</div>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 88f74ef..17f8c95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -22,7 +22,7 @@ info:
Bringing a new service on YARN today is not a simple experience. The APIs of existing frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) or writing a complex spec (for declarative frameworks). In addition to building critical building blocks inside YARN (as part of other efforts at link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a need for simplifying the user facing story for building services. Experience of projects like Apache Slider running real-life services like HBase, Storm, Accumulo, Solr etc, gives us some very good insights on how simplified APIs for services should look like.
- To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component apps to complex multi-component assemblies needing orchestration.
+ To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component service to complex multi-component assemblies needing orchestration.
We should also look at making this a unified REST based entry point for other important features like resource-profile management (link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), package-definitions' lifecycle-management and service-discovery (link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]). We also need to flesh out its relation to our present much lower level REST APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN for application-submission and management.
@@ -41,177 +41,177 @@ schemes:
- http
- https
# will be prefixed to all paths
-basePath: /services/v1/
+basePath: /ws/v1/
consumes:
- application/json
produces:
- application/json
paths:
- /applications:
+ /services:
get:
- summary: List of applications/services running in the cluster
- description: Get a list of all currently running applications (response includes a minimal projection of the application info). For more details do a GET on a specific application name.
+ summary: List of services running in the cluster
+ description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
responses:
200:
- description: An array of applications
+ description: An array of services
schema:
type: array
items:
- $ref: '#/definitions/Application'
+ $ref: '#/definitions/Service'
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
post:
- summary: Create an application/service
- description: Create an application. The request JSON is an Application object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the application creation request. There is no guarantee that the application will actually reach a RUNNING state. Resource availability and several other factors determines if the application will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the application and determine its state.
+ summary: Create a service
+ description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
parameters:
- - name: Application
+ - name: Service
in: body
- description: Application request object
+ description: Service request object
required: true
schema:
- $ref: '#/definitions/Application'
+ $ref: '#/definitions/Service'
responses:
202:
description: Request accepted
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
- /applications/{app_name}:
+ /services/{service_name}:
get:
- summary: Get an application/service details
- description: Return the details (including containers) of a running application
+ summary: Get service details
+ description: Return the details (including containers) of a running service
parameters:
- - name: app_name
+ - name: service_name
in: path
- description: Application name
+ description: Service name
required: true
type: string
responses:
200:
- description: An application object
+ description: a service object
schema:
type: object
items:
- $ref: '#/definitions/Application'
+ $ref: '#/definitions/Service'
examples:
- app_name: logsearch
+ service_name: logsearch
artifact:
id: logsearch:latest
type: docker
404:
- description: Application does not exist
+ description: Service does not exist
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
put:
- summary: Update an application/service or upgrade the binary version of the components of a running application
- description: Update the runtime properties of an application. As of now, only update of lifetime and number of instances (flexing) of the components of an application is supported. The PUT operation is also used to orchestrate an upgrade of the application containers to a newer version of their artifacts.
+ summary: Update a service or upgrade the binary version of the components of a running service
+ description: Update the runtime properties of a service. As of now, only update of lifetime and number of instances (flexing) of the components of a service is supported. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts.
parameters:
- - name: app_name
+ - name: service_name
in: path
- description: Application name
+ description: Service name
required: true
type: string
responses:
204:
description: Update or upgrade was successful
404:
- description: Application does not exist
+ description: Service does not exist
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
delete:
- summary: Destroy application/service
- description: Destroy an application and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet.
+ summary: Destroy service
+ description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet.
parameters:
- - name: app_name
+ - name: service_name
in: path
- description: Application name
+ description: Service name
required: true
type: string
responses:
204:
description: Destroy was successful
404:
- description: Application does not exist
+ description: Service does not exist
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
definitions:
- Application:
- description: An Application resource has the following attributes.
+ Service:
+ description: a service resource has the following attributes.
required:
- name
properties:
name:
type: string
- description: A unique application name. If Registry DNS is enabled, the max length is 63 characters.
+ description: A unique service name. If Registry DNS is enabled, the max length is 63 characters.
id:
type: string
- description: A unique application id.
+ description: A unique service id.
artifact:
- description: Artifact of single-component applications.
+ description: Artifact of single-component service.
$ref: '#/definitions/Artifact'
resource:
- description: Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.
+ description: Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.
$ref: '#/definitions/Resource'
launch_command:
type: string
- description: The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.
+ description: The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.
launch_time:
type: string
format: date
- description: The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
+ description: The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
number_of_containers:
type: integer
format: int64
- description: Number of containers for each app-component in the application. Each app-component can further override this app-level global default.
+ description: Number of containers for each component in the service. Each component can further override this service-level global default.
number_of_running_containers:
type: integer
format: int64
- description: In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.
+ description: In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.
lifetime:
type: integer
format: int64
- description: Life time (in seconds) of the application from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
+ description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
placement_policy:
- description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
+ description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
$ref: '#/definitions/PlacementPolicy'
components:
- description: Components of an application.
+ description: Components of a service.
type: array
items:
$ref: '#/definitions/Component'
configuration:
- description: Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.
+ description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.
$ref: '#/definitions/Configuration'
containers:
- description: Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.
+ description: Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.
type: array
items:
$ref: '#/definitions/Container'
state:
- description: State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.
- $ref: '#/definitions/ApplicationState'
+ description: State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.
+ $ref: '#/definitions/ServiceState'
quicklinks:
type: object
- description: A blob of key-value pairs of quicklinks to be exported for an application.
+ description: A blob of key-value pairs of quicklinks to be exported for a service.
additionalProperties:
type: string
queue:
type: string
- description: The YARN queue that this application should be submitted to.
+ description: The YARN queue that this service should be submitted to.
Resource:
description:
- Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.
+ Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
properties:
profile:
type: string
@@ -224,75 +224,75 @@ definitions:
type: string
description: Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.
PlacementPolicy:
- description: Placement policy of an instance of an application. This feature is in the works in YARN-4902.
+ description: Placement policy of an instance of a service. This feature is in the works in YARN-6592.
properties:
label:
type: string
- description: Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.
+ description: Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.
Artifact:
- description: Artifact of an application component. If not specified, component will just run the bare launch command and no artifact will be localized.
+ description: Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
required:
- id
properties:
id:
type: string
- description: Artifact id. Examples are package location uri for tarball based apps, image name for docker, name of application, etc.
+ description: Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.
type:
type: string
- description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For APPLICATION type, the application specified will be read and its components will be added into this application. The original component with artifact type APPLICATION will be removed (any properties specified in the original component will be ignored).
+ description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).
enum:
- DOCKER
- TARBALL
- - APPLICATION
+ - SERVICE
default: DOCKER
uri:
type: string
description: Artifact location to support multiple artifact stores (optional).
Component:
- description: One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.
+ description: One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
required:
- name
properties:
name:
type: string
- description: Name of the application component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
+ description: Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
dependencies:
type: array
items:
type: string
- description: An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.
+ description: An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.
readiness_check:
- description: Readiness check for this app-component.
+ description: Readiness check for this component.
$ref: '#/definitions/ReadinessCheck'
artifact:
- description: Artifact of the component (optional). If not specified, the application level global artifact takes effect.
+ description: Artifact of the component (optional). If not specified, the service level global artifact takes effect.
$ref: '#/definitions/Artifact'
launch_command:
type: string
description: The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).
resource:
- description: Resource of this component (optional). If not specified, the application level global resource takes effect.
+ description: Resource of this component (optional). If not specified, the service level global resource takes effect.
$ref: '#/definitions/Resource'
number_of_containers:
type: integer
format: int64
- description: Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.
+ description: Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.
run_privileged_container:
type: boolean
description: Run all containers of this component in privileged mode (YARN-4262).
placement_policy:
- description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.
+ description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.
$ref: '#/definitions/PlacementPolicy'
configuration:
- description: Config properties for this app-component.
+ description: Config properties for this component.
$ref: '#/definitions/Configuration'
quicklinks:
type: array
items:
type: string
- description: A list of quicklink keys defined at the application level, and to be resolved by this component.
+ description: A list of quicklink keys defined at the service level, and to be resolved by this component.
ReadinessCheck:
- description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.
+ description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.
required:
- type
properties:
@@ -308,28 +308,28 @@ definitions:
additionalProperties:
type: string
artifact:
- description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET
+ description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET
$ref: '#/definitions/Artifact'
Configuration:
- description: Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
+ description: Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
properties:
properties:
type: object
- description: A blob of key-value pairs of common application properties.
+ description: A blob of key-value pairs of common service properties.
additionalProperties:
type: string
env:
type: object
- description: A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.
+ description: A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.
additionalProperties:
type: string
files:
- description: Array of list of files that needs to be created and made available as volumes in the application component containers.
+ description: Array of list of files that needs to be created and made available as volumes in the service component containers.
type: array
items:
$ref: '#/definitions/ConfigFile'
ConfigFile:
- description: A config file that needs to be created and made available as a volume in an application component container.
+ description: A config file that needs to be created and made available as a volume in a service component container.
properties:
type:
type: string
@@ -352,11 +352,11 @@ definitions:
type: object
description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.
Container:
- description: An instance of a running application container.
+ description: An instance of a running service container.
properties:
id:
type: string
- description: Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.
+ description: Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.
launch_time:
type: string
format: date
@@ -371,7 +371,7 @@ definitions:
type: string
description: The bare node or host in which the container is running, e.g. cn008.example.com.
state:
- description: State of the container of an application.
+ description: State of the container of a service.
$ref: '#/definitions/ContainerState'
component_name:
type: string
@@ -385,12 +385,12 @@ definitions:
privileged_container:
type: boolean
description: Container running in privileged mode or not.
- ApplicationState:
- description: The current state of an application.
+ ServiceState:
+ description: The current state of a service.
properties:
state:
type: string
- description: enum of the state of the application
+ description: enum of the state of the service
enum:
- ACCEPTED
- STARTED
@@ -398,25 +398,26 @@ definitions:
- STOPPED
- FAILED
ContainerState:
- description: The current state of the container of an application.
+ description: The current state of the container of a service.
properties:
state:
type: string
description: enum of the state of the container
enum:
- INIT
+ - STARTED
- READY
- ApplicationStatus:
- description: The current status of a submitted application, returned as a response to the GET API.
+ ServiceStatus:
+ description: The current status of a submitted service, returned as a response to the GET API.
properties:
diagnostics:
type: string
- description: Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.
+ description: Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.
state:
- description: Application state.
- $ref: '#/definitions/ApplicationState'
+ description: Service state.
+ $ref: '#/definitions/ServiceState'
code:
type: integer
format: int32
- description: An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.
+ description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
deleted file mode 100644
index 9f15b7e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export SLIDER_VERSION=${project.version}
-export HDP_VERSION=${HDP_VERSION:-$SLIDER_VERSION}
-export SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
-export LIB_PARENT_DIR=`dirname $SCRIPT_DIR`
-export JAVA_HOME=${JAVA_HOME:-/usr/jdk64/jdk1.8.0_40}
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
-export REST_SERVICE_PORT=${REST_SERVICE_PORT:-9191}
-export APP_RUNAS_USER=${APP_RUNAS_USER:-root}
-export REST_SERVICE_LOG_DIR=${REST_SERVICE_LOG_DIR:-/tmp/}
-export JAVA_OPTS="-Xms256m -Xmx1024m -XX:+PrintGC -Xloggc:$REST_SERVICE_LOG_DIR/gc.log"
-$JAVA_HOME/bin/java $JAVA_OPTS -cp .:$HADOOP_CONF_DIR:$LIB_PARENT_DIR/services-api/*:$LIB_PARENT_DIR/slider/* -DREST_SERVICE_LOG_DIR=$REST_SERVICE_LOG_DIR -Dlog4j.configuration=log4j-server.properties -Dslider.libdir=$LIB_PARENT_DIR/slider org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp 1>>$REST_SERVICE_LOG_DIR/restservice-out.log 2>&1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..2814cca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <Match>
+ <Package name="org.apache.hadoop.yarn.proto" />
+ </Match>
+ <Match>
+ <class name="org.apache.hadoop.yarn.service.utils.ServiceApiUtil" />
+ <Bug pattern="MS_CANNOT_BE_FINAL" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
+ <Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" />
+ <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
+ </Match>
+ <Match>
+ <Package name="org.apache.hadoop.yarn.service.client.params"/>
+ <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
+ </Match>
+ <Match>
+ <Package name="org.apache.hadoop.yarn.service.client.params"/>
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.service.client.ServiceClient"/>
+ <Field name="registryClient" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC"/>
+ </Match>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties
new file mode 100644
index 0000000..58c8e27
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for Slider Application Master
+
+# Log rotation based on size (256MB) with a max of 20 backup files
+log4j.rootLogger=INFO, amlog
+log4j.threshhold=ALL
+log4j.appender.amlog=org.apache.log4j.RollingFileAppender
+log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
+log4j.appender.amlog.File=${LOG_DIR}/serviceam.log
+log4j.appender.amlog.MaxFileSize=256MB
+log4j.appender.amlog.MaxBackupIndex=20
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+# for debugging yarn-service framework
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
new file mode 100644
index 0000000..1f8a408
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -0,0 +1,408 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-services</artifactId>
+ <version>3.0.0-beta1-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-yarn-services-core</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Hadoop YARN Services Core</name>
+
+ <properties>
+ <!-- Needed for generating FindBugs warnings using parent pom -->
+ <yarn.basedir>${project.parent.basedir}</yarn.basedir>
+ </properties>
+
+ <build>
+ <!-- resources are filtered for dynamic updates. This gets build info in-->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+ <param>${basedir}/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/main/proto</directory>
+ <includes>
+ <include>ClientAMProtocol.proto</include>
+ </includes>
+ </source>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <environmentVariables>
+ <JAVA_HOME>${java.home}</JAVA_HOME>
+ </environmentVariables>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ <version>1.30</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-digester</groupId>
+ <artifactId>commons-digester</artifactId>
+ <version>1.8</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-servlets</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- service registry -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- Jersey and webapp support -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-guice</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.1</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.objenesis</groupId>
+ <artifactId>objenesis</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-easymock</artifactId>
+ <version>1.6.5</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>1.6.5</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.objenesis</groupId>
+ <artifactId>objenesis</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>1.16</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-annotations</artifactId>
+ <version>1.5.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+
+ <profiles>
+ <profile>
+ <id>dist</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <descriptorRefs>
+ <descriptorRef>hadoop-yarn-services-dist</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
new file mode 100644
index 0000000..516d23d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+
+import java.io.IOException;
+
+public interface ClientAMProtocol {
+ FlexComponentsResponseProto flexComponents(FlexComponentsRequestProto request)
+ throws IOException, YarnException;
+
+ GetStatusResponseProto getStatus(GetStatusRequestProto requestProto)
+ throws IOException, YarnException;
+
+ StopResponseProto stop(StopRequestProto requestProto)
+ throws IOException, YarnException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
new file mode 100644
index 0000000..8e4c34d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
+
+public class ClientAMService extends AbstractService
+ implements ClientAMProtocol {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ClientAMService.class);
+
+ private ServiceContext context;
+ private Server server;
+
+ private InetSocketAddress bindAddress;
+
+ public ClientAMService(ServiceContext context) {
+ super("Client AM Service");
+ this.context = context;
+ }
+
+ @Override protected void serviceStart() throws Exception {
+ Configuration conf = getConfig();
+ YarnRPC rpc = YarnRPC.create(conf);
+ InetSocketAddress address = new InetSocketAddress(0);
+ server = rpc.getServer(ClientAMProtocol.class, this, address, conf,
+ context.secretManager, 1);
+ server.start();
+
+ String nodeHostString =
+ System.getenv(ApplicationConstants.Environment.NM_HOST.name());
+
+ bindAddress = NetUtils.createSocketAddrForHost(nodeHostString,
+ server.getListenerAddress().getPort());
+
+ LOG.info("Instantiated ClientAMService at " + bindAddress);
+ super.serviceStart();
+ }
+
+ @Override protected void serviceStop() throws Exception {
+ if (server != null) {
+ server.stop();
+ }
+ super.serviceStop();
+ }
+
+ @Override public FlexComponentsResponseProto flexComponents(
+ FlexComponentsRequestProto request) throws IOException {
+ if (!request.getComponentsList().isEmpty()) {
+ for (ComponentCountProto component : request.getComponentsList()) {
+ ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
+ .setDesired(component.getNumberOfContainers());
+ context.scheduler.getDispatcher().getEventHandler().handle(event);
+ LOG.info("Flexing component {} to {}", component.getName(),
+ component.getNumberOfContainers());
+ }
+ }
+ return FlexComponentsResponseProto.newBuilder().build();
+ }
+
+ @Override
+ public GetStatusResponseProto getStatus(GetStatusRequestProto request)
+ throws IOException, YarnException {
+ String stat = ServiceApiUtil.jsonSerDeser.toJson(context.service);
+ return GetStatusResponseProto.newBuilder().setStatus(stat).build();
+ }
+
+ @Override
+ public StopResponseProto stop(StopRequestProto requestProto)
+ throws IOException, YarnException {
+ LOG.info("Stop the service.");
+ // Stop the service in 2 seconds delay to make sure this rpc call is completed.
+ // shutdown hook will be executed which will stop AM gracefully.
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(2000);
+ ExitUtil.terminate(0);
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted while stopping", e);
+ }
+ }
+ };
+ thread.start();
+ return StopResponseProto.newBuilder().build();
+ }
+
+ public InetSocketAddress getBindAddress() {
+ return bindAddress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
new file mode 100644
index 0000000..4743f28
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.service.component.Component;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.NODE_BLACKLIST_THRESHOLD;
+
+/**
+ * This tracks the container failures per node. If the failure counter exceeds
+ * the maxFailurePerNode limit, it'll blacklist that node.
+ *
+ */
+public class ContainerFailureTracker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerFailureTracker.class);
+
+ // Host -> num container failures
+ private Map<String, Integer> failureCountPerNode = new HashMap<>();
+ private Set<String> blackListedNodes = new HashSet<>();
+ private ServiceContext context;
+ private int maxFailurePerNode;
+ private Component component;
+
+ public ContainerFailureTracker(ServiceContext context, Component component) {
+ this.context = context;
+ this.component = component;
+ maxFailurePerNode = component.getComponentSpec().getConfiguration()
+ .getPropertyInt(NODE_BLACKLIST_THRESHOLD, 3);
+ }
+
+
+ public synchronized void incNodeFailure(String host) {
+ int num = 0;
+ if (failureCountPerNode.containsKey(host)) {
+ num = failureCountPerNode.get(host);
+ }
+ num++;
+ failureCountPerNode.put(host, num);
+
+ // black list the node if exceed max failure
+ if (num > maxFailurePerNode && !blackListedNodes.contains(host)) {
+ List<String> blacklists = new ArrayList<>();
+ blacklists.add(host);
+ blackListedNodes.add(host);
+ context.scheduler.getAmRMClient().updateBlacklist(blacklists, null);
+ LOG.info("[COMPONENT {}]: Failed {} times on this host, blacklisted {}."
+ + " Current list of blacklisted nodes: {}",
+ component.getName(), num, host, blackListedNodes);
+ }
+ }
+
+ public synchronized void resetContainerFailures() {
+ // reset container failure counter per node
+ failureCountPerNode.clear();
+ context.scheduler.getAmRMClient()
+ .updateBlacklist(null, new ArrayList<>(blackListedNodes));
+ LOG.info("[COMPONENT {}]: Clearing blacklisted nodes {} ",
+ component.getName(), blackListedNodes);
+ blackListedNodes.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
new file mode 100644
index 0000000..94dbc6e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.cache.LoadingCache;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
+public class ServiceContext {
+ public Service service = null;
+ public SliderFileSystem fs;
+ public String serviceHdfsDir = "";
+ public ApplicationAttemptId attemptId;
+ public LoadingCache<ConfigFile, Object> configCache;
+ public ServiceScheduler scheduler;
+ public ClientToAMTokenSecretManager secretManager;
+ public ClientAMService clientAMService;
+
+ public ServiceContext() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
new file mode 100644
index 0000000..e700961
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
+import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class ServiceMaster extends CompositeService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceMaster.class);
+
+ private static SliderAMArgs amArgs;
+ protected ServiceContext context;
+
+ public ServiceMaster(String name) {
+ super(name);
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ //TODO Deprecate slider conf, make sure works with yarn conf
+ printSystemEnv();
+ if (UserGroupInformation.isSecurityEnabled()) {
+ UserGroupInformation.setConfiguration(conf);
+ }
+ LOG.info("Login user is {}", UserGroupInformation.getLoginUser());
+
+ context = new ServiceContext();
+ Path appDir = getAppDir();
+ context.serviceHdfsDir = appDir.toString();
+ SliderFileSystem fs = new SliderFileSystem(conf);
+ context.fs = fs;
+ fs.setAppDir(appDir);
+ loadApplicationJson(context, fs);
+
+ ContainerId amContainerId = getAMContainerId();
+
+ ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId();
+ LOG.info("Service AppAttemptId: " + attemptId);
+ context.attemptId = attemptId;
+
+ // configure AM to wait forever for RM
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
+ conf.unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
+
+ DefaultMetricsSystem.initialize("ServiceAppMaster");
+
+ context.secretManager = new ClientToAMTokenSecretManager(attemptId, null);
+ ClientAMService clientAMService = new ClientAMService(context);
+ context.clientAMService = clientAMService;
+ addService(clientAMService);
+
+ ServiceScheduler scheduler = createServiceScheduler(context);
+ addService(scheduler);
+ context.scheduler = scheduler;
+
+ ServiceMonitor monitor = new ServiceMonitor("Service Monitor", context);
+ addService(monitor);
+
+ super.serviceInit(conf);
+ }
+
+ protected ContainerId getAMContainerId() throws BadClusterStateException {
+ return ContainerId.fromString(SliderUtils.mandatoryEnvVariable(
+ ApplicationConstants.Environment.CONTAINER_ID.name()));
+ }
+
+ protected Path getAppDir() {
+ return new Path(amArgs.getAppDefPath()).getParent();
+ }
+
+ protected ServiceScheduler createServiceScheduler(ServiceContext context)
+ throws IOException, YarnException {
+ return new ServiceScheduler(context);
+ }
+
+ protected void loadApplicationJson(ServiceContext context,
+ SliderFileSystem fs) throws IOException {
+ context.service = ServiceApiUtil
+ .loadServiceFrom(fs, new Path(amArgs.getAppDefPath()));
+ LOG.info(context.service.toString());
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ LOG.info("Stopping app master");
+ super.serviceStop();
+ }
+
+ private void printSystemEnv() {
+ for (Map.Entry<String, String> envs : System.getenv().entrySet()) {
+ LOG.info("{} = {}", envs.getKey(), envs.getValue());
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
+ amArgs = new SliderAMArgs(args);
+ amArgs.parse();
+ try {
+ ServiceMaster serviceMaster = new ServiceMaster("Service Master");
+ ShutdownHookManager.get()
+ .addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30);
+ YarnConfiguration conf = new YarnConfiguration();
+ new GenericOptionsParser(conf, args);
+ serviceMaster.init(conf);
+ serviceMaster.start();
+ } catch (Throwable t) {
+ LOG.error("Error starting service master", t);
+ ExitUtil.terminate(1, "Error starting service master");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
new file mode 100644
index 0000000..9fc886e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+@Metrics(context = "yarn-native-service")
+public class ServiceMetrics implements MetricsSource {
+
+ @Metric("containers requested")
+ public MutableGaugeInt containersRequested;
+
+ @Metric("anti-affinity containers pending")
+ public MutableGaugeInt pendingAAContainers;
+
+ @Metric("containers running")
+ public MutableGaugeInt containersRunning;
+
+ @Metric("containers ready")
+ public MutableGaugeInt containersReady;
+
+ @Metric("containers desired")
+ public MutableGaugeInt containersDesired;
+
+ @Metric("containers succeeded")
+ public MutableGaugeInt containersSucceeded;
+
+ @Metric("containers failed")
+ public MutableGaugeInt containersFailed;
+
+ @Metric("containers preempted")
+ public MutableGaugeInt containersPreempted;
+
+ @Metric("containers surplus")
+ public MutableGaugeInt surplusContainers;
+
+ @Metric("containers failed due to disk failure")
+ public MutableGaugeInt containersDiskFailure;
+
+ protected final MetricsRegistry registry;
+
+ public ServiceMetrics(MetricsInfo metricsInfo) {
+ registry = new MetricsRegistry(metricsInfo);
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ registry.snapshot(collector.addRecord(registry.info()), all);
+ }
+
+ public static ServiceMetrics register(String name, String description) {
+ ServiceMetrics metrics = new ServiceMetrics(info(name, description));
+ DefaultMetricsSystem.instance().register(name, description, metrics);
+ return metrics;
+ }
+
+ public void tag(String name, String description, String value) {
+ registry.tag(name, description, value);
+ }
+
+ @Override public String toString() {
+ return "ServiceMetrics{"
+ + "containersRequested=" + containersRequested.value()
+ + ", pendingAAContainers=" + pendingAAContainers.value()
+ + ", containersRunning=" + containersRunning.value()
+ + ", containersDesired=" + containersDesired.value()
+ + ", containersSucceeded=" + containersSucceeded.value()
+ + ", containersFailed=" + containersFailed.value()
+ + ", containersPreempted=" + containersPreempted.value()
+ + ", surplusContainers=" + surplusContainers.value() + '}';
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/58] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in
yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/541b64c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/541b64c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/541b64c7
Branch: refs/heads/yarn-native-services
Commit: 541b64c7985db425c5ce8bd5f71a2d5980809708
Parents: b9ca13e
Author: Gour Saha <go...@apache.org>
Authored: Thu Sep 7 23:25:31 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:15 2017 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 4 +-
hadoop-project/src/site/site.xml | 10 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 7 +-
...RN-Simplified-V1-API-Layer-For-Services.yaml | 11 +-
.../hadoop/yarn/service/ServiceMaster.java | 10 +-
.../hadoop/yarn/service/client/ServiceCLI.java | 5 +-
.../yarn/service/client/ServiceClient.java | 20 +-
.../client/params/AbstractActionArgs.java | 33 +-
.../AbstractClusterBuildingActionArgs.java | 22 +-
.../service/client/params/ActionBuildArgs.java | 5 +
.../service/client/params/ActionCreateArgs.java | 5 +
.../client/params/ActionDependencyArgs.java | 4 +-
.../service/client/params/ActionExamples.java | 26 +
.../service/client/params/ActionUpdateArgs.java | 7 +-
.../yarn/service/client/params/Arguments.java | 9 +-
.../yarn/service/client/params/ClientArgs.java | 10 -
.../yarn/service/client/params/CommonArgs.java | 28 +-
.../service/client/params/ServiceAMArgs.java | 50 ++
.../client/params/ServiceAMCreateAction.java | 37 ++
.../service/client/params/SliderAMArgs.java | 57 --
.../client/params/SliderAMCreateAction.java | 73 ---
.../service/client/params/SliderActions.java | 9 +-
.../ServiceTimelinePublisher.java | 5 +-
.../client/TestBuildExternalComponents.java | 6 +-
.../yarn/service/client/TestServiceCLI.java | 30 +-
.../hadoop/yarn/service/conf/examples/app.json | 2 +-
.../registry/client/api/RegistryConstants.java | 2 +-
.../src/site/markdown/YarnCommands.md | 56 +-
.../native-services/NativeServicesAPI.md | 606 -------------------
.../native-services/NativeServicesDiscovery.md | 144 -----
.../native-services/NativeServicesIntro.md | 107 ----
.../src/site/markdown/yarn-service/Concepts.md | 77 +++
.../src/site/markdown/yarn-service/Overview.md | 58 ++
.../site/markdown/yarn-service/QuickStart.md | 218 +++++++
.../markdown/yarn-service/ServiceDiscovery.md | 150 +++++
.../markdown/yarn-service/YarnServiceAPI.md | 592 ++++++++++++++++++
36 files changed, 1363 insertions(+), 1132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3cb9568..d18c9a8 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -143,10 +143,10 @@
<!-- the version of Hadoop declared in the version resources; can be overridden
so that Hadoop 3.x can declare itself a 2.x artifact. -->
<declared.hadoop.version>${project.version}</declared.hadoop.version>
-
+
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>1.16</snakeyaml.version>
- <jcommander.version>1.30</jcommander.version>
+ <jcommander.version>1.48</jcommander.version>
</properties>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index ad893a2..57cff9a 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -155,10 +155,12 @@
<item name="Timeline Service V.2" href="hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html#Timeline_Service_v.2_REST_API"/>
</menu>
- <menu name="YARN Native Services" inherit="top">
- <item name="Introduction" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesIntro.html"/>
- <item name="Native Services API" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesAPI.html"/>
- <item name="Native Services Discovery" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesDiscovery.html"/>
+ <menu name="YARN Service" inherit="top">
+ <item name="Overview" href="hadoop-yarn/hadoop-yarn-site/yarn-service/Overview.html"/>
+ <item name="QuickStart" href="hadoop-yarn/hadoop-yarn-site/yarn-service/QuickStart.html"/>
+ <item name="Concepts" href="hadoop-yarn/hadoop-yarn-site/yarn-service/Concepts.html"/>
+ <item name="Yarn Service API" href="hadoop-yarn/hadoop-yarn-site/yarn-service/YarnServiceAPI.html"/>
+ <item name="Service Discovery" href="hadoop-yarn/hadoop-yarn-site/yarn-service/ServiceDiscovery.html"/>
</menu>
<menu name="Hadoop Compatible File Systems" inherit="top">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 1db585e..05d892b 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -31,7 +31,7 @@ function hadoop_usage
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
- hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
+ hadoop_add_subcommand "apiserver" daemon "run the api-server for deploying/managing services on YARN"
hadoop_add_subcommand "application" client "prints application(s) report/kill application"
hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report"
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
@@ -50,10 +50,9 @@ function hadoop_usage
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
- hadoop_add_subcommand "service" "run a service"
- hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon"
+ hadoop_add_subcommand "service" client "deploy/manage a service on YARN"
+ hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
hadoop_add_subcommand "top" client "view cluster information"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index b084be7..b9b5b3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -48,17 +48,16 @@ info:
url: http://www.apache.org/licenses/LICENSE-2.0.html
# the domain of the service
host: host.mycompany.com
+port: 9191(default)
# array of all schemes that your API supports
schemes:
- http
-# will be prefixed to all paths
-basePath: /ws/v1/
consumes:
- application/json
produces:
- application/json
paths:
- /services/version:
+ /ws/v1/services/version:
get:
summary: Get current version of the API server.
description: Get current version of the API server.
@@ -66,7 +65,7 @@ paths:
200:
description: Successful request
- /services:
+ /ws/v1/services:
get:
summary: (TBD) List of services running in the cluster.
description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
@@ -103,7 +102,7 @@ paths:
schema:
$ref: '#/definitions/ServiceStatus'
- /services/{service_name}:
+ /ws/v1/services/{service_name}:
put:
summary: Update a service or upgrade the binary version of the components of a running service
description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service.
@@ -174,7 +173,7 @@ paths:
description: Unexpected error
schema:
$ref: '#/definitions/ServiceStatus'
- /services/{service_name}/components/{component_name}:
+ /ws/v1/services/{service_name}/components/{component_name}:
put:
summary: Flex a component's number of instances.
description: Set a component's desired number of instanes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index e700961..2abdae1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
+import org.apache.hadoop.yarn.service.client.params.ServiceAMArgs;
import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
@@ -51,7 +51,7 @@ public class ServiceMaster extends CompositeService {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceMaster.class);
- private static SliderAMArgs amArgs;
+ private static ServiceAMArgs amArgs;
protected ServiceContext context;
public ServiceMaster(String name) {
@@ -108,7 +108,7 @@ public class ServiceMaster extends CompositeService {
}
protected Path getAppDir() {
- return new Path(amArgs.getAppDefPath()).getParent();
+ return new Path(amArgs.getServiceDefPath()).getParent();
}
protected ServiceScheduler createServiceScheduler(ServiceContext context)
@@ -119,7 +119,7 @@ public class ServiceMaster extends CompositeService {
protected void loadApplicationJson(ServiceContext context,
SliderFileSystem fs) throws IOException {
context.service = ServiceApiUtil
- .loadServiceFrom(fs, new Path(amArgs.getAppDefPath()));
+ .loadServiceFrom(fs, new Path(amArgs.getServiceDefPath()));
LOG.info(context.service.toString());
}
@@ -138,7 +138,7 @@ public class ServiceMaster extends CompositeService {
public static void main(String[] args) throws Exception {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
- amArgs = new SliderAMArgs(args);
+ amArgs = new ServiceAMArgs(args);
amArgs.parse();
try {
ServiceMaster serviceMaster = new ServiceMaster("Service Master");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
index cb27021..928c06f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
@@ -18,11 +18,12 @@
package org.apache.hadoop.yarn.service.client;
+import com.beust.jcommander.ParameterException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,7 +102,7 @@ public class ServiceCLI {
ClientArgs clientArgs = new ClientArgs(args);
try {
clientArgs.parse();
- } catch (BadCommandArgumentsException e) {
+ } catch (ParameterException | SliderException e) {
System.err.println(e.getMessage());
System.exit(-1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index c3a2752..bfddc44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -158,7 +158,7 @@ public class ServiceClient extends CompositeService
private Service loadAppJsonFromLocalFS(
AbstractClusterBuildingActionArgs args) throws IOException {
- File file = args.getAppDef();
+ File file = args.getFile();
Path filePath = new Path(file.getAbsolutePath());
LOG.info("Loading app json from: " + filePath);
Service service = jsonSerDeser
@@ -166,7 +166,6 @@ public class ServiceClient extends CompositeService
if (args.lifetime > 0) {
service.setLifetime(args.lifetime);
}
- service.setName(args.getClusterName());
return service;
}
@@ -407,7 +406,8 @@ public class ServiceClient extends CompositeService
public int actionDestroy(String serviceName) throws Exception {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
- verifyNoLiveAppInRM(serviceName, "Destroy");
+ verifyNoLiveAppInRM(serviceName, "destroy");
+
Path appDir = fs.buildClusterDirPath(serviceName);
FileSystem fileSystem = fs.getFileSystem();
// remove from the appId cache
@@ -498,9 +498,15 @@ public class ServiceClient extends CompositeService
request.setApplicationStates(liveStates);
List<ApplicationReport> reports = yarnClient.getApplications(request);
if (!reports.isEmpty()) {
- throw new YarnException(
- "Failed to " + action + " service, as " + serviceName
- + " already exists.");
+ String message = "";
+ if (action.equals("destroy")) {
+ message = "Failed to destroy service " + serviceName
+ + ", because it is still running.";
+ } else {
+ message = "Failed to " + action + " service " + serviceName
+ + ", because it already exists.";
+ }
+ throw new YarnException(message);
}
}
@@ -592,7 +598,7 @@ public class ServiceClient extends CompositeService
CLI.add(ServiceMaster.class.getCanonicalName());
CLI.add(ACTION_CREATE, serviceName);
//TODO debugAM CLI.add(Arguments.ARG_DEBUG)
- CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, serviceName + ".json"));
+ CLI.add(Arguments.ARG_SERVICE_DEF_PATH, new Path(appRootDir, serviceName + ".json"));
// pass the registry binding
CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
index ea3bb0a..8b2d93e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
@@ -41,22 +41,10 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
}
/**
- * URI/binding to the filesystem
- */
- @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
- description = "Filesystem Binding")
- public String filesystemBinding;
-
- @Parameter(names = {ARG_BASE_PATH},
- description = "Service base path on the filesystem",
- converter = PathArgumentConverter.class)
- public Path basePath;
-
- /**
* This is the default parameter
*/
@Parameter
- public final List<String> parameters = new ArrayList<>();
+ public List<String> parameters = new ArrayList<>();
/**
* get the name: relies on arg 1 being the cluster name in all operations
@@ -76,29 +64,22 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
*/
- @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
- public final List<String> definitions = new ArrayList<>();
+ @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions", hidden = true)
+ public List<String> definitions = new ArrayList<>();
/**
* System properties
*/
@Parameter(names = {ARG_SYSPROP}, arity = 1,
description = "system properties in the form name value" +
- " These are set after the JVM is started.")
- public final List<String> sysprops = new ArrayList<>(0);
+ " These are set after the JVM is started.",
+ hidden = true)
+ public List<String> sysprops = new ArrayList<>(0);
- @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
- description = "Binding (usually hostname:port) of the YARN resource manager")
- public String manager;
-
-
- @Parameter(names = ARG_DEBUG, description = "Debug mode")
+ @Parameter(names = ARG_DEBUG, description = "Debug mode", hidden = true)
public boolean debug = false;
- @Parameter(names = {ARG_HELP}, description = "Help", help = true)
- public boolean help = false;
-
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
index 3a3a19a..d215436 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -19,13 +19,8 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.io.File;
-import java.util.List;
-import java.util.Map;
/**
* Abstract Action to build things; shares args across build and
@@ -33,12 +28,12 @@ import java.util.Map;
*/
public abstract class AbstractClusterBuildingActionArgs
extends AbstractActionArgs {
- @Parameter(names = {ARG_APPDEF},
- description = "Template service definition file in JSON format.")
- public File appDef;
+ @Parameter(names = { ARG_FILE, ARG_FILE_SHORT }, required = true,
+ description = "The path to the service definition file in JSON format.")
+ public File file;
- public File getAppDef() {
- return appDef;
+ public File getFile() {
+ return file;
}
@Parameter(names = {
@@ -48,11 +43,4 @@ public abstract class AbstractClusterBuildingActionArgs
@Parameter(names = {
ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
public long lifetime;
-
- @ParametersDelegate
- public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
-
- @ParametersDelegate
- public OptionArgsDelegate optionsDelegate =
- new OptionArgsDelegate();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
index c2ff545..cf61525 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
@@ -28,4 +28,9 @@ public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
public String getActionName() {
return SliderActions.ACTION_BUILD;
}
+
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
index eecffb6..c43f4fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
@@ -29,5 +29,10 @@ public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
public String getActionName() {
return SliderActions.ACTION_CREATE;
}
+
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
index 51e07c9..699e6ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
@@ -32,8 +32,8 @@ public class ActionDependencyArgs extends AbstractActionArgs {
return SliderActions.ACTION_DEPENDENCY;
}
- @Parameter(names = { ARG_UPLOAD },
- description = "Upload AM and agent libraries to HDFS for this client")
+ @Parameter(names = { ARG_UPLOAD }, required = true,
+ description = "Upload AM libraries to HDFS for this client version")
public boolean upload;
@Parameter(names = { ARG_OVERWRITE },
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
new file mode 100644
index 0000000..e489e17
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_EXAMPLES},
+ commandDescription = SliderActions.DESCRIBE_ACTION_EXAMPLES)
+public class ActionExamples {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
index e310f45..00af69a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
@@ -18,12 +18,17 @@
package org.apache.hadoop.yarn.service.client.params;
+import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
-public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
+public class ActionUpdateArgs extends AbstractActionArgs {
+
+ @Parameter(names = {
+ ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
+ public long lifetime;
@Override
public String getActionName() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
index 204149b..911440d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -27,7 +27,8 @@ package org.apache.hadoop.yarn.service.client.params;
*/
public interface Arguments {
- String ARG_APPDEF = "--appdef";
+ String ARG_FILE = "--file";
+ String ARG_FILE_SHORT = "-f";
String ARG_BASE_PATH = "--basepath";
String ARG_COMPONENT = "--component";
String ARG_COMPONENT_SHORT = "--comp";
@@ -95,9 +96,7 @@ public interface Arguments {
RIGHT PLACE IN THE LIST
*/
- /**
- * server: URI for the cluster
- */
- String ARG_CLUSTER_URI = "-cluster-uri";
+ // Tha path in hdfs to be read by Service AM
+ String ARG_SERVICE_DEF_PATH = "-cluster-uri";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
index 09cae24..f479cd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
@@ -85,16 +85,6 @@ public class ClientArgs extends CommonArgs {
public void applyDefinitions(Configuration conf) throws
BadCommandArgumentsException {
super.applyDefinitions(conf);
- //RM
- if (getManager() != null) {
- log.debug("Setting RM to {}", getManager());
- conf.set(YarnConfiguration.RM_ADDRESS, getManager());
- }
- if (getBasePath() != null) {
- log.debug("Setting basePath to {}", getBasePath());
- conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH,
- getBasePath().toString());
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
index e1197ea..a5638a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
@@ -156,12 +156,8 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
try {
commander.parse(args);
} catch (ParameterException e) {
- throw new BadCommandArgumentsException(e, "%s in %s",
- e.toString(),
- (args != null
- ? (SliderUtils.join(args,
- " ", false))
- : "[]"));
+ commander.usage(args[0]);
+ throw e;
}
//now copy back to this class some of the attributes that are common to all
//actions
@@ -251,31 +247,11 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
applyDefinitions(definitionMap, conf);
}
-
- /**
- * If the Filesystem binding was provided, it overrides anything in
- * the configuration
- * @param conf configuration
- */
- public void applyFileSystemBinding(Configuration conf) {
- ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf);
- }
-
public boolean isDebug() {
return coreAction.debug;
}
- public String getFilesystemBinding() {
- return coreAction.filesystemBinding;
- }
-
- public Path getBasePath() { return coreAction.basePath; }
-
- public String getManager() {
- return coreAction.manager;
- }
-
public String getAction() {
return commander.getParsedCommand();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
new file mode 100644
index 0000000..800bb24
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.yarn.service.client.params;
+
+/**
+ * Parameters sent by the Client to the AM
+ */
+public class ServiceAMArgs extends CommonArgs {
+
+ ServiceAMCreateAction createAction = new ServiceAMCreateAction();
+
+ public ServiceAMArgs(String[] args) {
+ super(args);
+ }
+
+ @Override
+ protected void addActionArguments() {
+ addActions(createAction);
+ }
+
+ // This is the path in hdfs to the service definition JSON file
+ public String getServiceDefPath() {
+ return createAction.serviceDefPath;
+ }
+
+ /**
+ * Am binding is simple: there is only one action
+ */
+ @Override
+ public void applyAction() {
+ bindCoreAction(createAction);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
new file mode 100644
index 0000000..384e8ba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_CREATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class ServiceAMCreateAction extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CREATE;
+ }
+
+ @Parameter(names = ARG_SERVICE_DEF_PATH,
+ description = "Path to the service definition JSON file", required = true)
+ public String serviceDefPath;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
deleted file mode 100644
index 1c38213..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Parameters sent by the Client to the AM
- */
-public class SliderAMArgs extends CommonArgs {
-
- SliderAMCreateAction createAction = new SliderAMCreateAction();
-
- public SliderAMArgs(String[] args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
- addActions(createAction);
- }
-
- public String getImage() {
- return createAction.image;
- }
-
- /**
- * This is the URI in the FS to the Slider cluster; the conf file (and any
- * other cluster-specifics) can be picked up here
- */
- public String getAppDefPath() {
- return createAction.sliderClusterURI;
- }
-
- /**
- * Am binding is simple: there is only one action
- */
- @Override
- public void applyAction() {
- bindCoreAction(createAction);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
deleted file mode 100644
index a446665..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-import java.io.File;
-
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class SliderAMCreateAction extends AbstractActionArgs implements
- LaunchArgsAccessor {
-
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-
- @Parameter(names = ARG_IMAGE, description = "image", required = false)
- public String image;
-
- /**
- * This is the URI in the FS to the Slider cluster; the conf file (and any
- * other cluster-specifics) can be picked up here
- */
- @Parameter(names = ARG_CLUSTER_URI,
- description = "URI to the Slider cluster", required = true)
- public String sliderClusterURI;
-
- @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
-
- @Override
- public String getRmAddress() {
- return launchArgs.getRmAddress();
- }
-
- @Override
- public int getWaittime() {
- return launchArgs.getWaittime();
- }
-
- @Override
- public void setWaittime(int waittime) {
- launchArgs.setWaittime(waittime);
- }
-
- @Override
- public File getOutputFile() {
- return launchArgs.getOutputFile();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
index 1b2a92d..fa05f2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
@@ -19,9 +19,7 @@
package org.apache.hadoop.yarn.service.client.params;
/**
- * Actions.
- * Only some of these are supported by specific Slider Services; they
- * are listed here to ensure the names are consistent
+ * Actions by client
*/
public interface SliderActions {
String ACTION_BUILD = "build";
@@ -32,11 +30,11 @@ public interface SliderActions {
String ACTION_UPGRADE = "upgrade";
String ACTION_DESTROY = "destroy";
String ACTION_EXISTS = "exists";
+ String ACTION_EXAMPLES = "examples";
String ACTION_FLEX = "flex";
String ACTION_STOP = "stop";
String ACTION_HELP = "help";
String ACTION_INSTALL_KEYTAB = "install-keytab";
- String ACTION_KDIAG = "kdiag";
String ACTION_KEYTAB = "keytab";
String ACTION_LIST = "list";
@@ -50,7 +48,7 @@ public interface SliderActions {
String DESCRIBE_ACTION_BUILD =
"Build a service specification, but do not start it";
String DESCRIBE_ACTION_CREATE =
- "Build and start a service, it's equivalent to first invoke build and then start";
+ "Create a service, it's equivalent to first invoke build and then start";
String DESCRIBE_ACTION_DEPENDENCY =
"Yarn service framework dependency (libraries) management";
String DESCRIBE_ACTION_UPDATE =
@@ -61,6 +59,7 @@ public interface SliderActions {
"Destroy a stopped service, service must be stopped first before destroying.";
String DESCRIBE_ACTION_EXISTS =
"Probe for a service running";
+ String DESCRIBE_ACTION_EXAMPLES = "Run an example service on YARN";
String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
String DESCRIBE_ACTION_FREEZE =
"Stop a running service";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index 9e7d8e8..cfe7c1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.api.records.Configuration;
import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.slf4j.Logger;
@@ -96,8 +97,8 @@ public class ServiceTimelinePublisher extends CompositeService {
// create info keys
Map<String, Object> entityInfos = new HashMap<String, Object>();
entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName());
-// entityInfos.put(ServiceTimelineMetricsConstants.STATE,
-// service.getState().toString());
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+ ServiceState.STARTED.toString());
entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
currentTimeMillis);
entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
index 93a15cc..0498ebf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
@@ -36,7 +36,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
+import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
@@ -56,12 +56,12 @@ public class TestBuildExternalComponents {
}
}
- // 1. Build the appDef and store on fs
+ // 1. Build the def file and store on fs
// 2. check component names
private void buildAndCheckComponents(String appName, String appDef,
SliderFileSystem sfs, Set<String> names) throws Throwable {
String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
+ { "build", ARG_FILE, ExampleAppJson.resourceName(appDef) };
ClientArgs clientArgs = new ClientArgs(args);
clientArgs.parse();
ServiceCLI cli = new ServiceCLI() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index c53ee2b..f22d487 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
@@ -43,7 +42,7 @@ import java.util.List;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
+import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
import static org.mockito.Mockito.*;
@@ -54,9 +53,9 @@ public class TestServiceCLI {
private ServiceCLI cli;
private SliderFileSystem fs;
- private void buildApp(String appName, String appDef) throws Throwable {
+ private void buildApp(String appDef) throws Throwable {
String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
+ { "build", ARG_FILE, ExampleAppJson.resourceName(appDef)};
ClientArgs clientArgs = new ClientArgs(args);
clientArgs.parse();
cli.exec(clientArgs);
@@ -115,34 +114,35 @@ public class TestServiceCLI {
// Test flex components count are persisted.
@Test
public void testFlexComponents() throws Throwable {
- buildApp("service-1", ExampleAppJson.APP_JSON);
-
- checkCompCount("master", 1L);
+ String serviceName = "app-1";
+ buildApp(ExampleAppJson.APP_JSON);
+ checkCompCount("master",serviceName, 1L);
// increase by 2
- String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"};
+ String[] flexUpArgs = {"flex", serviceName, "--component", "master" , "+2"};
ClientArgs clientArgs = new ClientArgs(flexUpArgs);
clientArgs.parse();
cli.exec(clientArgs);
- checkCompCount("master", 3L);
+ checkCompCount("master", serviceName, 3L);
// decrease by 1
- String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"};
+ String[] flexDownArgs = {"flex", serviceName, "--component", "master", "-1"};
clientArgs = new ClientArgs(flexDownArgs);
clientArgs.parse();
cli.exec(clientArgs);
- checkCompCount("master", 2L);
+ checkCompCount("master", serviceName, 2L);
- String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"};
+ String[] flexAbsoluteArgs = {"flex", serviceName, "--component", "master", "10"};
clientArgs = new ClientArgs(flexAbsoluteArgs);
clientArgs.parse();
cli.exec(clientArgs);
- checkCompCount("master", 10L);
+ checkCompCount("master", serviceName, 10L);
}
- private void checkCompCount(String compName, long count) throws IOException {
+ private void checkCompCount(String compName, String serviceName, long count)
+ throws IOException {
List<Component> components =
- ServiceApiUtil.getComponents(fs, "service-1");
+ ServiceApiUtil.getComponents(fs, serviceName);
for (Component component : components) {
if (component.getName().equals(compName)) {
Assert.assertEquals(count, component.getNumberOfContainers().longValue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
index 12b51e4..43d3c39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json
@@ -1,5 +1,5 @@
{
- "name": "service-1",
+ "name": "app-1",
"id" : "application_1503358878042_0011",
"lifetime": "3600",
"launch_command": "sleep 3600",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
index e66a761..0006dfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
@@ -95,7 +95,7 @@ public interface RegistryConstants {
/**
* Default DNS port number.
*/
- int DEFAULT_DNS_PORT = 53;
+ int DEFAULT_DNS_PORT = 5353;
/**
* DNSSEC Enabled?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index fa33b8b..ca46c8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -22,7 +22,7 @@ Overview
YARN commands are invoked by the bin/yarn script. Running the yarn script without any arguments prints the description for all commands.
-Usage: `yarn [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [COMMAND_OPTIONS]`
+Usage: `yarn [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [SUB_COMMAND] [COMMAND_OPTIONS]`
YARN has an option parsing framework that employs parsing generic options as well as running classes.
@@ -69,6 +69,56 @@ Usage: `yarn applicationattempt [options] `
prints applicationattempt(s) report
+### `service`
+Usage `yarn service [sub-command] [service-name] [options]`
+
+#### `SUB_COMMAND`
+
+* `build`: Build a service with its specifications, but do not start it.
+ ```
+ Usage: yarn service build --file [file]
+ ```
+
+ | COMMAND\_OPTIONS | Description |
+ |:---- |:---- |
+ | --file or -f | The local path to the service definition file |
+
+* `create`: create a service, it's equivalent to first invoke build and then start.
+ ```
+ Usage: yarn service create --file [file]
+ ```
+ | COMMAND\_OPTIONS | Description |
+ |:---- |:---- |
+ | --file or -f | The local path to the service definition file |
+
+* `dependency`: Yarn service framework dependency (libraries) management.
+ ```
+ Usage: yarn service dependency [options]
+ ```
+ | COMMAND\_OPTIONS | Description |
+ |:---- |:---- |
+ | --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. |
+
+* `destroy`: Destroy a stopped service, service must be stopped first before destroying.
+ ```
+ Usage: yarn service destroy [service-name]
+ ```
+* `flex`: Flex a service's component by increasing or decreasing the number of containers.
+ ```
+ Usage: yarn service flex [service-name] --component [component-name] [count]
+ ```
+ | COMMAND\_OPTIONS | Description |
+ |:---- |:---- |
+ | --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.|
+* `status`: Get the status of a service.
+ ```
+ Usage: yarn service status [service-name]
+ ```
+* `start`: Start a service with pre-built specification or a previously stopped service.
+ ```
+ Usage: yarn service start [service-name]
+ ```
+
### `classpath`
Usage: `yarn classpath [--glob |--jar <path> |-h |--help]`
@@ -273,6 +323,10 @@ Usage: `yarn timelineserver`
Start the TimeLineServer
+### apiserver
+Usage: `yarn apiserver`
+Start the API-server for deploying/managing services on YARN
+
Files
-----
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/58] [abbrv] hadoop git commit: Revert "YARN-6626. Embed REST API
service into RM. Contributed by Eric Yang"
Posted by ji...@apache.org.
Revert "YARN-6626. Embed REST API service into RM. Contributed by Eric Yang"
This reverts commit 63d1084e9781e0fee876916190b69f6242dd00e4.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/673c0db4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/673c0db4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/673c0db4
Branch: refs/heads/yarn-native-services
Commit: 673c0db43cfa4a0e9e7a21c5ea31660ecf61b8c3
Parents: 9e677fa
Author: Jian He <ji...@apache.org>
Authored: Wed Oct 4 23:39:27 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:17 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 2 --
.../hadoop/yarn/service/webapp/ApiServer.java | 7 ----
.../hadoop-yarn-services-core/pom.xml | 6 ++++
.../yarn/service/api/records/Artifact.java | 8 -----
.../yarn/service/api/records/ConfigFile.java | 8 -----
.../service/api/records/ReadinessCheck.java | 8 -----
.../src/main/resources/yarn-default.xml | 8 -----
.../server/resourcemanager/webapp/RMWebApp.java | 19 -----------
.../site/markdown/yarn-service/QuickStart.md | 34 +++-----------------
9 files changed, 11 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4799137..640e86e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -336,8 +336,6 @@ public class YarnConfiguration extends Configuration {
public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
+ "webapp.ui2.war-file-path";
- public static final String YARN_API_SERVICES_ENABLE = "yarn."
- + "webapp.api-service.enable";
public static final String RM_RESOURCE_TRACKER_ADDRESS =
RM_PREFIX + "resource-tracker.address";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 89b020d..e8286ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.yarn.service.webapp;
-import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.VersionInfo;
@@ -58,12 +57,6 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
@Singleton
@Path(CONTEXT_ROOT)
public class ApiServer {
-
- @Inject
- public ApiServer(Configuration conf) {
- super();
- }
-
private static final Logger LOG =
LoggerFactory.getLogger(ApiServer.class);
private static Configuration YARN_CONFIG = new YarnConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index 205a64d..fb07edc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -245,6 +245,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
index ce062cc..4328c6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
@@ -23,9 +23,6 @@ import io.swagger.annotations.ApiModelProperty;
import java.io.Serializable;
import java.util.Objects;
-import javax.xml.bind.annotation.XmlEnum;
-import javax.xml.bind.annotation.XmlType;
-
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
@@ -45,11 +42,6 @@ public class Artifact implements Serializable {
private String id = null;
- /**
- * Artifact Type. DOCKER, TARBALL or SERVICE
- **/
- @XmlType(name = "artifact_type")
- @XmlEnum
public enum TypeEnum {
DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
index fb088e2..c1502c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -26,10 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlRootElement;
-import javax.xml.bind.annotation.XmlType;
-
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
@@ -48,11 +45,6 @@ import java.util.Objects;
public class ConfigFile implements Serializable {
private static final long serialVersionUID = -7009402089417704612L;
- /**
- * Config Type. XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported.
- **/
- @XmlType(name = "config_type")
- @XmlEnum
public enum TypeEnum {
XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
"TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
index b25828f..1a25a4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -25,9 +25,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
-import javax.xml.bind.annotation.XmlEnum;
-import javax.xml.bind.annotation.XmlType;
-
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -46,11 +43,6 @@ import org.apache.hadoop.classification.InterfaceStability;
public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L;
- /**
- * Type. HTTP and PORT
- **/
- @XmlType(name = "type")
- @XmlEnum
public enum TypeEnum {
HTTP("HTTP"),
PORT("PORT");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e90d0f2..f4b2e61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -212,14 +212,6 @@
</property>
<property>
- <description>
- Enable services rest api on ResourceManager.
- </description>
- <name>yarn.webapp.api-service.enable</name>
- <value>false</value>
- </property>
-
- <property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>${yarn.resourcemanager.hostname}:8031</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index bee9354..4e36665 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -22,8 +22,6 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.RMHAUtils;
@@ -40,12 +38,8 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
*/
public class RMWebApp extends WebApp implements YarnWebParams {
- private static final Log LOG =
- LogFactory.getLog(RMWebApp.class.getName());
private final ResourceManager rm;
private boolean standby = false;
- private final static String APISERVER =
- "org.apache.hadoop.yarn.service.webapp.ApiServer";
public RMWebApp(ResourceManager rm) {
this.rm = rm;
@@ -59,19 +53,6 @@ public class RMWebApp extends WebApp implements YarnWebParams {
bind(RMWebApp.class).toInstance(this);
if (rm != null) {
- boolean enableServiceApi = rm.getConfig()
- .getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false);
- if (enableServiceApi) {
- try {
- // Use reflection here to load ApiServer class,
- // this is done to avoid creating cyclic dependency
- // between maven projects.
- Class<?> apiServer = Class.forName(APISERVER);
- bind(apiServer);
- } catch (ClassNotFoundException e) {
- LOG.warn("ApiServer REST API is not activated.");
- }
- }
bind(ResourceManager.class).toInstance(rm);
}
route("/", RmController.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index afcd5f5..15df0cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -101,41 +101,17 @@ yarn service destroy ${SERVICE_NAME}
```
## Manage services on YARN via REST API
-
-YARN API Server REST API can be activated in two modes: embedded or standalone.
-
-### Start Embedded API-Server as part of ResourceManager
-For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
-
-```
- <property>
- <description>
- Enable services rest api on ResourceManager.
- </description>
- <name>yarn.webapp.api-service.enable</name>
- <value>true</value>
- </property>
-```
-
-Services can be deployed on YARN through the ResourceManager web endpoint.
-
-### Start Standalone API-Server for deploying services on YARN
-API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP.
+Below steps walk you through deploying services on YARN via REST API.
+ Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
+### Start API-Server for deploying services on YARN
+API server is the service that sits in front of YARN ResourceManager and lets users submit their API specs via HTTP.
```
yarn --daemon start apiserver
```
The above command starts the API Server on the localhost at port 9191 by default.
-Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
-
### Deploy a service
-
-POST the aforementioned example service definition to the ResourceManager api-server endpoint:
-```
-POST http://localhost:8088/ws/v1/services
-```
-
-Or standalone API server:
+POST the aforementioned example service definition to the api-server endpoint:
```
POST http://localhost:9191/ws/v1/services
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
deleted file mode 100644
index fe8cce8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * Methods to aid in config, both in the Configuration class and
- * with other parts of setting up Slider-initated processes.
- *
- * Some of the methods take an argument of a map iterable for their sources; this allows
- * the same method
- */
-public class ConfigHelper {
- private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
-
- /**
- * Set an entire map full of values
- *
- * @param config config to patch
- * @param map map of data
- * @param origin origin data
- */
- public static void addConfigMap(Configuration config,
- Map<String, String> map,
- String origin) throws BadConfigException {
- addConfigMap(config, map.entrySet(), origin);
- }
-
- /**
- * Set an entire map full of values
- *
- * @param config config to patch
- * @param map map of data
- * @param origin origin data
- */
- public static void addConfigMap(Configuration config,
- Iterable<Map.Entry<String, String>> map,
- String origin) throws BadConfigException {
- for (Map.Entry<String, String> mapEntry : map) {
- String key = mapEntry.getKey();
- String value = mapEntry.getValue();
- if (value == null) {
- throw new BadConfigException("Null value for property " + key);
- }
- config.set(key, value, origin);
- }
- }
-
- /**
- * Convert to an XML string
- * @param conf configuration
- * @return conf
- * @throws IOException
- */
- public static String toXml(Configuration conf) throws IOException {
- StringWriter writer = new StringWriter();
- conf.writeXml(writer);
- return writer.toString();
- }
-
-
- /**
- * Register a resource as a default resource.
- * Do not attempt to use this unless you understand that the
- * order in which default resources are loaded affects the outcome,
- * and that subclasses of Configuration often register new default
- * resources
- * @param resource the resource name
- * @return the URL or null
- */
- public static URL registerDefaultResource(String resource) {
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- Configuration.addDefaultResource(resource);
- }
- return resURL;
- }
-
- /**
- * Load a configuration from a resource on this classpath.
- * If the resource is not found, an empty configuration is returned
- * @param resource the resource name
- * @return the loaded configuration.
- */
- public static Configuration loadFromResource(String resource) {
- Configuration conf = new Configuration(false);
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- log.debug("loaded resources from {}", resURL);
- conf.addResource(resource);
- } else{
- log.debug("failed to find {} on the classpath", resource);
- }
- return conf;
-
- }
-
- /**
- * Get the URL to a resource, null if not on the CP
- * @param resource resource to look for
- * @return the URL or null
- */
- public static URL getResourceUrl(String resource) {
- return ConfigHelper.class.getClassLoader()
- .getResource(resource);
- }
-
- /**
- * This goes through the keyset of one configuration and retrieves each value
- * from a value source -a different or the same configuration. This triggers
- * the property resolution process of the value, resolving any variables against
- * in-config or inherited configurations
- * @param keysource source of keys
- * @param valuesource the source of values
- * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
- */
- public static Configuration resolveConfiguration(
- Iterable<Map.Entry<String, String>> keysource,
- Configuration valuesource) {
- Configuration result = new Configuration(false);
- for (Map.Entry<String, String> entry : keysource) {
- String key = entry.getKey();
- String value = valuesource.get(key);
- Preconditions.checkState(value != null,
- "no reference for \"%s\" in values", key);
- result.set(key, value);
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
deleted file mode 100644
index a969be9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigUtils.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class ConfigUtils {
- public static final String TEMPLATE_FILE = "template.file";
-
- public static String replaceProps(Map<String, String> config, String content) {
- Map<String, String> tokens = new HashMap<>();
- for (Entry<String, String> entry : config.entrySet()) {
- tokens.put("${" + entry.getKey() + "}", entry.getValue());
- tokens.put("{{" + entry.getKey() + "}}", entry.getValue());
- }
- String value = content;
- for (Map.Entry<String,String> token : tokens.entrySet()) {
- value = value.replaceAll(Pattern.quote(token.getKey()),
- Matcher.quoteReplacement(token.getValue()));
- }
- return value;
- }
-
- public static Map<String, String> replacePropsInConfig(
- Map<String, String> config, Map<String, String> env) {
- Map<String, String> tokens = new HashMap<>();
- for (Entry<String, String> entry : env.entrySet()) {
- tokens.put("${" + entry.getKey() + "}", entry.getValue());
- }
- Map<String, String> newConfig = new HashMap<>();
- for (Entry<String, String> entry : config.entrySet()) {
- String value = entry.getValue();
- for (Map.Entry<String,String> token : tokens.entrySet()) {
- value = value.replaceAll(Pattern.quote(token.getKey()),
- Matcher.quoteReplacement(token.getValue()));
- }
- newConfig.put(entry.getKey(), entry.getValue());
- }
- return newConfig;
- }
-
- public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
- Map<String, String> config, SliderFileSystem fileSystem,
- String clusterName, String fileName) throws IOException {
- if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
- return;
- }
- Path templateFile = null;
- if (config.containsKey(TEMPLATE_FILE)) {
- templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(clusterName,
- config.get(TEMPLATE_FILE));
- }
- if (!fileSystem.isFile(templateFile)) {
- throw new IOException("config specified template file " + config
- .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
- }
- }
- if (templateFile == null && fileName != null) {
- templateFile = fileSystem.buildResourcePath(fileName);
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(clusterName,
- fileName);
- }
- }
- if (fileSystem.isFile(templateFile)) {
- config.put("content", fileSystem.cat(templateFile));
- } else {
- config.put("content", "");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
deleted file mode 100644
index fa3b402..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ /dev/null
@@ -1,521 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.util.Records;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class CoreFileSystem {
- private static final Logger
- log = LoggerFactory.getLogger(CoreFileSystem.class);
-
- private static final String UTF_8 = "UTF-8";
-
- protected final FileSystem fileSystem;
- protected final Configuration configuration;
-
- public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
- Preconditions.checkNotNull(fileSystem,
- "Cannot create a CoreFileSystem with a null FileSystem");
- Preconditions.checkNotNull(configuration,
- "Cannot create a CoreFileSystem with a null Configuration");
- this.fileSystem = fileSystem;
- this.configuration = configuration;
- }
-
- public CoreFileSystem(Configuration configuration) throws IOException {
- Preconditions.checkNotNull(configuration,
- "Cannot create a CoreFileSystem with a null Configuration");
- this.fileSystem = FileSystem.get(configuration);
- this.configuration = configuration;
- }
-
- /**
- * Get the temp path for this cluster
- * @param clustername name of the cluster
- * @return path for temp files (is not purged)
- */
- public Path getTempPathForCluster(String clustername) {
- Path clusterDir = buildClusterDirPath(clustername);
- return new Path(clusterDir, YarnServiceConstants.TMP_DIR_PREFIX);
- }
-
- /**
- * Returns the underlying FileSystem for this object.
- *
- * @return filesystem
- */
- public FileSystem getFileSystem() {
- return fileSystem;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("CoreFileSystem{");
- sb.append("fileSystem=").append(fileSystem.getUri());
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Build up the path string for a cluster instance -no attempt to
- * create the directory is made
- *
- * @param clustername name of the cluster
- * @return the path for persistent data
- */
- public Path buildClusterDirPath(String clustername) {
- Preconditions.checkNotNull(clustername);
- Path path = getBaseApplicationPath();
- return new Path(path, YarnServiceConstants.SERVICES_DIRECTORY + "/" + clustername);
- }
-
-
- /**
- * Build up the path string for keytab install location -no attempt to
- * create the directory is made
- *
- * @return the path for keytab
- */
- public Path buildKeytabInstallationDirPath(String keytabFolder) {
- Preconditions.checkNotNull(keytabFolder);
- Path path = getBaseApplicationPath();
- return new Path(path, YarnServiceConstants.KEYTAB_DIR + "/" + keytabFolder);
- }
-
- /**
- * Build up the path string for keytab install location -no attempt to
- * create the directory is made
- *
- * @return the path for keytab installation location
- */
- public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
- Path homePath = getHomeDirectory();
- Path baseKeytabDir;
- if (keytabDir != null) {
- baseKeytabDir = new Path(homePath, keytabDir);
- } else {
- baseKeytabDir = new Path(buildClusterDirPath(clusterName),
- YarnServiceConstants.KEYTAB_DIR);
- }
- return keytabName == null ? baseKeytabDir :
- new Path(baseKeytabDir, keytabName);
- }
-
- /**
- * Build up the path string for resource install location -no attempt to
- * create the directory is made
- *
- * @return the path for resource
- */
- public Path buildResourcePath(String resourceFolder) {
- Preconditions.checkNotNull(resourceFolder);
- Path path = getBaseApplicationPath();
- return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + resourceFolder);
- }
-
- /**
- * Build up the path string for resource install location -no attempt to
- * create the directory is made
- *
- * @return the path for resource
- */
- public Path buildResourcePath(String dirName, String fileName) {
- Preconditions.checkNotNull(dirName);
- Preconditions.checkNotNull(fileName);
- Path path = getBaseApplicationPath();
- return new Path(path, YarnServiceConstants.RESOURCE_DIR + "/" + dirName + "/" + fileName);
- }
-
- /**
- * Create a directory with the given permissions.
- *
- * @param dir directory
- * @param clusterPerms cluster permissions
- * @throws IOException IO problem
- * @throws BadClusterStateException any cluster state problem
- */
- @SuppressWarnings("deprecation")
- public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
- IOException,
- BadClusterStateException {
- if (fileSystem.isFile(dir)) {
- // HADOOP-9361 shows some filesystems don't correctly fail here
- throw new BadClusterStateException(
- "Cannot create a directory over a file %s", dir);
- }
- log.debug("mkdir {} with perms {}", dir, clusterPerms);
- //no mask whatoever
- fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
- fileSystem.mkdirs(dir, clusterPerms);
- //and force set it anyway just to make sure
- fileSystem.setPermission(dir, clusterPerms);
- }
-
- /**
- * Verify that the cluster directory is not present
- *
- * @param clustername name of the cluster
- * @param clusterDirectory actual directory to look for
- * @throws IOException trouble with FS
- * @throws SliderException If the directory exists
- */
- public void verifyClusterDirectoryNonexistent(String clustername,
- Path clusterDirectory)
- throws IOException, SliderException {
- if (fileSystem.exists(clusterDirectory)) {
- throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
- clusterDirectory);
- }
- }
- /**
- * Verify that the given directory is not present
- *
- * @param clusterDirectory actual directory to look for
- * @throws IOException trouble with FS
- * @throws SliderException If the directory exists
- */
- public void verifyDirectoryNonexistent(Path clusterDirectory) throws
- IOException,
- SliderException {
- if (fileSystem.exists(clusterDirectory)) {
-
- log.error("Dir {} exists: {}",
- clusterDirectory,
- listFSDir(clusterDirectory));
- throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
- clusterDirectory);
- }
- }
-
- /**
- * Verify that a user has write access to a directory.
- * It does this by creating then deleting a temp file
- *
- * @param dirPath actual directory to look for
- * @throws FileNotFoundException file not found
- * @throws IOException trouble with FS
- * @throws BadClusterStateException if the directory is not writeable
- */
- public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
- SliderException {
- verifyPathExists(dirPath);
- Path tempFile = new Path(dirPath, "tmp-file-for-checks");
- try {
- FSDataOutputStream out ;
- out = fileSystem.create(tempFile, true);
- IOUtils.closeStream(out);
- fileSystem.delete(tempFile, false);
- } catch (IOException e) {
- log.warn("Failed to create file {}: {}", tempFile, e);
- throw new BadClusterStateException(e,
- "Unable to write to directory %s : %s", dirPath, e.toString());
- }
- }
-
- /**
- * Verify that a path exists
- * @param path path to check
- * @throws FileNotFoundException file not found
- * @throws IOException trouble with FS
- */
- public void verifyPathExists(Path path) throws IOException {
- if (!fileSystem.exists(path)) {
- throw new FileNotFoundException(path.toString());
- }
- }
-
- /**
- * Verify that a path exists
- * @param path path to check
- * @throws FileNotFoundException file not found or is not a file
- * @throws IOException trouble with FS
- */
- public void verifyFileExists(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
-
- if (!status.isFile()) {
- throw new FileNotFoundException("Not a file: " + path.toString());
- }
- }
-
- /**
- * Given a path, check if it exists and is a file
- *
- * @param path
- * absolute path to the file to check
- * @return true if and only if path exists and is a file, false for all other
- * reasons including if file check throws IOException
- */
- public boolean isFile(Path path) {
- boolean isFile = false;
- try {
- FileStatus status = fileSystem.getFileStatus(path);
- if (status.isFile()) {
- isFile = true;
- }
- } catch (IOException e) {
- // ignore, isFile is already set to false
- }
- return isFile;
- }
-
- /**
- * Get the base path
- *
- * @return the base path optionally configured by
- * {@link YarnServiceConf#YARN_SERVICE_BASE_PATH}
- */
- public Path getBaseApplicationPath() {
- String configuredBasePath = configuration
- .get(YarnServiceConf.YARN_SERVICE_BASE_PATH,
- getHomeDirectory() + "/" + YarnServiceConstants.SERVICE_BASE_DIRECTORY);
- return new Path(configuredBasePath);
- }
-
- /**
- * Get slider dependency parent dir in HDFS
- *
- * @return the parent dir path of slider.tar.gz in HDFS
- */
- public Path getDependencyPath() {
- String parentDir = YarnServiceConstants.DEPENDENCY_DIR;
- return new Path(String.format(parentDir, VersionInfo.getVersion()));
- }
-
- /**
- * Get slider.tar.gz absolute filepath in HDFS
- *
- * @return the absolute path to slider.tar.gz in HDFS
- */
- public Path getDependencyTarGzip() {
- Path dependencyLibAmPath = getDependencyPath();
- Path dependencyLibTarGzip = new Path(
- dependencyLibAmPath.toUri().toString(),
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
- + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
- return dependencyLibTarGzip;
- }
-
- public Path getHomeDirectory() {
- return fileSystem.getHomeDirectory();
- }
-
- /**
- * Create an AM resource from the
- *
- * @param destPath dest path in filesystem
- * @param resourceType resource type
- * @return the local resource for AM
- */
- public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
- FileStatus destStatus = fileSystem.getFileStatus(destPath);
- LocalResource amResource = Records.newRecord(LocalResource.class);
- amResource.setType(resourceType);
- // Set visibility of the resource
- // Setting to most private option
- amResource.setVisibility(LocalResourceVisibility.APPLICATION);
- // Set the resource to be copied over
- amResource.setResource(
- URL.fromPath(fileSystem.resolvePath(destStatus.getPath())));
- // Set timestamp and length of file so that the framework
- // can do basic sanity checks for the local resource
- // after it has been copied over to ensure it is the same
- // resource the client intended to use with the application
- amResource.setTimestamp(destStatus.getModificationTime());
- amResource.setSize(destStatus.getLen());
- return amResource;
- }
-
- /**
- * Register all files under a fs path as a directory to push out
- *
- * @param srcDir src dir
- * @param destRelativeDir dest dir (no trailing /)
- * @return the map of entries
- */
- public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
- //now register each of the files in the directory to be
- //copied to the destination
- FileStatus[] fileset = fileSystem.listStatus(srcDir);
- Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>(fileset.length);
- for (FileStatus entry : fileset) {
-
- LocalResource resource = createAmResource(entry.getPath(),
- LocalResourceType.FILE);
- String relativePath = destRelativeDir + "/" + entry.getPath().getName();
- localResources.put(relativePath, resource);
- }
- return localResources;
- }
-
- /**
- * Submit a JAR containing a specific class, returning
- * the resource to be mapped in
- *
- * @param clazz class to look for
- * @param subdir subdirectory (expected to end in a "/")
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
- throws IOException, SliderException {
- File localFile = SliderUtils.findContainingJarOrFail(clazz);
- return submitFile(localFile, tempPath, subdir, jarName);
- }
-
- /**
- * Submit a local file to the filesystem references by the instance's cluster
- * filesystem
- *
- * @param localFile filename
- * @param subdir subdirectory (expected to end in a "/")
- * @param destFileName destination filename
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName)
- throws IOException {
- Path src = new Path(localFile.toString());
- Path subdirPath = new Path(tempPath, subdir);
- fileSystem.mkdirs(subdirPath);
- Path destPath = new Path(subdirPath, destFileName);
- log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath);
-
- fileSystem.copyFromLocalFile(false, true, src, destPath);
-
- // Set the type of resource - file or archive
- // archives are untarred at destination
- // we don't need the jar file to be untarred for now
- return createAmResource(destPath, LocalResourceType.FILE);
- }
-
- /**
- * Submit the AM tar.gz resource referenced by the instance's cluster
- * filesystem. Also, update the providerResources object with the new
- * resource.
- *
- * @param providerResources
- * the provider resource map to be updated
- * @throws IOException
- * trouble copying to HDFS
- */
- public void submitTarGzipAndUpdate(
- Map<String, LocalResource> providerResources) throws IOException,
- BadClusterStateException {
- Path dependencyLibTarGzip = getDependencyTarGzip();
- LocalResource lc = createAmResource(dependencyLibTarGzip,
- LocalResourceType.ARCHIVE);
- providerResources.put(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK, lc);
- }
-
- public void copyLocalFileToHdfs(File localPath,
- Path destPath, FsPermission fp)
- throws IOException {
- if (localPath == null || destPath == null) {
- throw new IOException("Either localPath or destPath is null");
- }
- fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
- "000");
- fileSystem.mkdirs(destPath.getParent(), fp);
- log.info("Copying file {} to {}", localPath.toURI(),
- fileSystem.getScheme() + ":/" + destPath.toUri());
-
- fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
- destPath);
- // set file permissions of the destPath
- fileSystem.setPermission(destPath, fp);
- }
-
- public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
- throws IOException {
- if (hdfsPath == null || destFile == null) {
- throw new IOException("Either hdfsPath or destPath is null");
- }
- log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
-
- Path destPath = new Path(destFile.getPath());
- fileSystem.copyToLocalFile(hdfsPath, destPath);
- }
-
- /**
- * list entries in a filesystem directory
- *
- * @param path directory
- * @return a listing, one to a line
- * @throws IOException
- */
- public String listFSDir(Path path) throws IOException {
- FileStatus[] stats = fileSystem.listStatus(path);
- StringBuilder builder = new StringBuilder();
- for (FileStatus stat : stats) {
- builder.append(stat.getPath().toString())
- .append("\t")
- .append(stat.getLen())
- .append("\n");
- }
- return builder.toString();
- }
-
- public String cat(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
- byte[] b = new byte[(int) status.getLen()];
- FSDataInputStream in = null;
- try {
- in = fileSystem.open(path);
- int count = in.read(b);
- return new String(b, 0, count, UTF_8);
- } finally {
- IOUtils.closeStream(in);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
deleted file mode 100644
index 6fadfd3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Duration.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import java.io.Closeable;
-
-/**
- * A duration in milliseconds. This class can be used
- * to count time, and to be polled to see if a time limit has
- * passed.
- */
-public class Duration implements Closeable {
- public long start, finish;
- public final long limit;
-
- /**
- * Create a duration instance with a limit of 0
- */
- public Duration() {
- this(0);
- }
-
- /**
- * Create a duration with a limit specified in millis
- * @param limit duration in milliseconds
- */
- public Duration(long limit) {
- this.limit = limit;
- }
-
- /**
- * Start
- * @return self
- */
- public Duration start() {
- start = now();
- return this;
- }
-
- /**
- * The close operation relays to {@link #finish()}.
- * Implementing it allows Duration instances to be automatically
- * finish()'d in Java7 try blocks for when used in measuring durations.
- */
- @Override
- public final void close() {
- finish();
- }
-
- public void finish() {
- finish = now();
- }
-
- protected long now() {
- return System.nanoTime()/1000000;
- }
-
- public long getInterval() {
- return finish - start;
- }
-
- /**
- * return true if the limit has been exceeded
- * @return true if a limit was set and the current time
- * exceeds it.
- */
- public boolean getLimitExceeded() {
- return limit >= 0 && ((now() - start) > limit);
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("Duration");
- if (finish >= start) {
- builder.append(" finished at ").append(getInterval()).append(" millis;");
- } else {
- if (start > 0) {
- builder.append(" started but not yet finished;");
- } else {
- builder.append(" unstarted;");
- }
- }
- if (limit > 0) {
- builder.append(" limit: ").append(limit).append(" millis");
- if (getLimitExceeded()) {
- builder.append(" - exceeded");
- }
- }
- return builder.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
deleted file mode 100644
index 7b22e3e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-import org.codehaus.jackson.map.SerializationConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-/**
- * Support for marshalling objects to and from JSON.
- * This class is NOT thread safe; it constructs an object mapper
- * as an instance field.
- * @param <T>
- */
-public class JsonSerDeser<T> {
-
- private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class);
- private static final String UTF_8 = "UTF-8";
-
- private final Class<T> classType;
- private final ObjectMapper mapper;
-
- /**
- * Create an instance bound to a specific type
- * @param classType class type
- */
- public JsonSerDeser(Class<T> classType) {
- this.classType = classType;
- this.mapper = new ObjectMapper();
- mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- public JsonSerDeser(Class<T> classType, PropertyNamingStrategy namingStrategy) {
- this(classType);
- mapper.setPropertyNamingStrategy(namingStrategy);
- }
-
- /**
- * Convert from JSON
- * @param json input
- * @return the parsed JSON
- * @throws IOException IO
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- public T fromJson(String json)
- throws IOException, JsonParseException, JsonMappingException {
- try {
- return mapper.readValue(json, classType);
- } catch (IOException e) {
- log.error("Exception while parsing json : " + e + "\n" + json, e);
- throw e;
- }
- }
-
- /**
- * Convert from a JSON file
- * @param jsonFile input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- public T fromFile(File jsonFile)
- throws IOException, JsonParseException, JsonMappingException {
- File absoluteFile = jsonFile.getAbsoluteFile();
- try {
- return mapper.readValue(absoluteFile, classType);
- } catch (IOException e) {
- log.error("Exception while parsing json file {}", absoluteFile, e);
- throw e;
- }
- }
-
- /**
- * Convert from a JSON file
- * @param resource input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- public T fromResource(String resource)
- throws IOException, JsonParseException, JsonMappingException {
- try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
- if (resStream == null) {
- throw new FileNotFoundException(resource);
- }
- return (T) (mapper.readValue(resStream, classType));
- } catch (IOException e) {
- log.error("Exception while parsing json resource {}", resource, e);
- throw e;
- }
- }
-
- /**
- * Convert from an input stream, closing the stream afterwards.
- * @param stream
- * @return the parsed JSON
- * @throws IOException IO problems
- */
- public T fromStream(InputStream stream) throws IOException {
- try {
- return (T) (mapper.readValue(stream, classType));
- } catch (IOException e) {
- log.error("Exception while parsing json input stream", e);
- throw e;
- } finally {
- IOUtils.closeStream(stream);
- }
- }
-
- /**
- * clone by converting to JSON and back again.
- * This is much less efficient than any Java clone process.
- * @param instance instance to duplicate
- * @return a new instance
- * @throws IOException problems.
- */
- public T fromInstance(T instance) throws IOException {
- return fromJson(toJson(instance));
- }
-
- /**
- * Deserialize from a byte array
- * @param b
- * @return the deserialized value
- * @throws IOException parse problems
- */
- public T fromBytes(byte[] b) throws IOException {
- String json = new String(b, 0, b.length, UTF_8);
- return fromJson(json);
- }
-
- /**
- * Load from a Hadoop filesystem
- * @param fs filesystem
- * @param path path
- * @return a loaded CD
- * @throws IOException IO problems
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public T load(FileSystem fs, Path path)
- throws IOException, JsonParseException, JsonMappingException {
- FileStatus status = fs.getFileStatus(path);
- long len = status.getLen();
- byte[] b = new byte[(int) len];
- FSDataInputStream dataInputStream = fs.open(path);
- int count = dataInputStream.read(b);
- if (count != len) {
- throw new EOFException("Read of " + path +" finished prematurely");
- }
- return fromBytes(b);
- }
-
-
- /**
- * Save to a hadoop filesystem
- * @param fs filesystem
- * @param path path
- * @param instance instance to save
- * @param overwrite should any existing file be overwritten
- * @throws IOException IO exception
- */
- public void save(FileSystem fs, Path path, T instance,
- boolean overwrite) throws
- IOException {
- FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
- writeJsonAsBytes(instance, dataOutputStream);
- }
-
- /**
- * Save an instance to a file
- * @param instance instance to save
- * @param file file
- * @throws IOException
- */
- public void save(T instance, File file) throws
- IOException {
- writeJsonAsBytes(instance, new FileOutputStream(file.getAbsoluteFile()));
- }
-
- /**
- * Write the json as bytes -then close the file
- * @param dataOutputStream an outout stream that will always be closed
- * @throws IOException on any failure
- */
- private void writeJsonAsBytes(T instance,
- OutputStream dataOutputStream) throws IOException {
- try {
- String json = toJson(instance);
- byte[] b = json.getBytes(UTF_8);
- dataOutputStream.write(b);
- dataOutputStream.flush();
- dataOutputStream.close();
- } finally {
- IOUtils.closeStream(dataOutputStream);
- }
- }
-
- /**
- * Convert an object to a JSON string
- * @param instance instance to convert
- * @return a JSON string description
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public String toJson(T instance) throws IOException,
- JsonGenerationException,
- JsonMappingException {
- mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
- return mapper.writeValueAsString(instance);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
deleted file mode 100644
index c0712c3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
+++ /dev/null
@@ -1,680 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SaslPropertiesResolver;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.Cipher;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.lang.reflect.InvocationTargetException;
-import java.net.InetAddress;
-import java.security.NoSuchAlgorithmException;
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import static org.apache.hadoop.security.UserGroupInformation.*;
-import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
-
-/**
- * Kerberos diagnostics
- * At some point this may move to hadoop core, so please keep use of slider
- * methods and classes to ~0.
- *
- * This operation expands some of the diagnostic output of the security code,
- * but not all. For completeness
- *
- * Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
- * Set the log level for {@code org.apache.hadoop.security=DEBUG}
- */
-public class KerberosDiags implements Closeable {
-
- private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class);
- public static final String KRB5_CCNAME = "KRB5CCNAME";
- public static final String JAVA_SECURITY_KRB5_CONF
- = "java.security.krb5.conf";
- public static final String JAVA_SECURITY_KRB5_REALM
- = "java.security.krb5.realm";
- public static final String SUN_SECURITY_KRB5_DEBUG
- = "sun.security.krb5.debug";
- public static final String SUN_SECURITY_SPNEGO_DEBUG
- = "sun.security.spnego.debug";
- public static final String SUN_SECURITY_JAAS_FILE
- = "java.security.auth.login.config";
- public static final String KERBEROS_KINIT_COMMAND
- = "hadoop.kerberos.kinit.command";
- public static final String HADOOP_AUTHENTICATION_IS_DISABLED
- = "Hadoop authentication is disabled";
- public static final String UNSET = "(unset)";
- public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
-
- private final Configuration conf;
- private final List<String> services;
- private final PrintStream out;
- private final File keytab;
- private final String principal;
- private final long minKeyLength;
- private final boolean securityRequired;
-
- public static final String CAT_JVM = "JVM";
- public static final String CAT_JAAS = "JAAS";
- public static final String CAT_CONFIG = "CONFIG";
- public static final String CAT_LOGIN = "LOGIN";
- public static final String CAT_KERBEROS = "KERBEROS";
- public static final String CAT_SASL = "SASL";
-
- @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
- public KerberosDiags(Configuration conf,
- PrintStream out,
- List<String> services,
- File keytab,
- String principal,
- long minKeyLength,
- boolean securityRequired) {
- this.conf = conf;
- this.services = services;
- this.keytab = keytab;
- this.principal = principal;
- this.out = out;
- this.minKeyLength = minKeyLength;
- this.securityRequired = securityRequired;
- }
-
- @Override
- public void close() throws IOException {
- flush();
- }
-
- /**
- * Execute diagnostics.
- * <p>
- * Things it would be nice if UGI made accessible
- * <ol>
- * <li>A way to enable JAAS debug programatically</li>
- * <li>Access to the TGT</li>
- * </ol>
- * @return true if security was enabled and all probes were successful
- * @throws KerberosDiagsFailure explicitly raised failure
- * @throws Exception other security problems
- */
- @SuppressWarnings("deprecation")
- public boolean execute() throws Exception {
-
- title("Kerberos Diagnostics scan at %s",
- new Date(System.currentTimeMillis()));
-
- // check that the machine has a name
- println("Hostname: %s",
- InetAddress.getLocalHost().getCanonicalHostName());
-
- // Fail fast on a JVM without JCE installed.
- validateKeyLength();
-
- // look at realm
- println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName());
- printDefaultRealm();
-
- title("System Properties");
- for (String prop : new String[]{
- JAVA_SECURITY_KRB5_CONF,
- JAVA_SECURITY_KRB5_REALM,
- SUN_SECURITY_KRB5_DEBUG,
- SUN_SECURITY_SPNEGO_DEBUG,
- SUN_SECURITY_JAAS_FILE
- }) {
- printSysprop(prop);
- }
-
- title("Environment Variables");
- for (String env : new String[]{
- "HADOOP_JAAS_DEBUG",
- KRB5_CCNAME,
- "HADOOP_USER_NAME",
- "HADOOP_PROXY_USER",
- HADOOP_TOKEN_FILE_LOCATION,
- }) {
- printEnv(env);
- }
-
- for (String prop : new String[]{
- KERBEROS_KINIT_COMMAND,
- HADOOP_SECURITY_AUTHENTICATION,
- HADOOP_SECURITY_AUTHORIZATION,
- "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6
- "hadoop.security.dns.interface", // not in 2.6
- "hadoop.security.dns.nameserver", // not in 2.6
- HADOOP_RPC_PROTECTION,
- HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
- HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX,
- HADOOP_SECURITY_GROUP_MAPPING,
- "hadoop.security.impersonation.provider.class", // not in 2.6
- "dfs.data.transfer.protection" // HDFS
- }) {
- printConfOpt(prop);
- }
-
- // check that authentication is enabled
- if (SecurityUtil.getAuthenticationMethod(conf)
- .equals(AuthenticationMethod.SIMPLE)) {
- println(HADOOP_AUTHENTICATION_IS_DISABLED);
- failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED);
- // no security, skip rest of test
- return false;
- }
-
- validateKrb5File();
- validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS);
- validateSasl("dfs.data.transfer.saslproperties.resolver.class");
- validateKinitExecutable();
- validateJAAS();
- // now the big test: login, then try again
- boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG);
- boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG);
- try {
- title("Logging in");
-
- if (keytab != null) {
- dumpKeytab(keytab);
- loginFromKeytab();
- } else {
- UserGroupInformation loginUser = getLoginUser();
- dumpUGI("Log in user", loginUser);
- validateUGI("Login user", loginUser);
- println("Ticket based login: %b", isLoginTicketBased());
- println("Keytab based login: %b", isLoginKeytabBased());
- }
-
- return true;
- } finally {
- // restore original system properties
- System.setProperty(SUN_SECURITY_KRB5_DEBUG,
- Boolean.toString(krb5Debug));
- System.setProperty(SUN_SECURITY_SPNEGO_DEBUG,
- Boolean.toString(spnegoDebug));
- }
- }
-
- /**
- * Fail fast on a JVM without JCE installed.
- *
- * This is a recurrent problem
- * (that is: it keeps creeping back with JVM updates);
- * a fast failure is the best tactic
- * @throws NoSuchAlgorithmException
- */
-
- protected void validateKeyLength() throws NoSuchAlgorithmException {
- int aesLen = Cipher.getMaxAllowedKeyLength("AES");
- println("Maximum AES encryption key length %d bits", aesLen);
- failif (aesLen < minKeyLength,
- CAT_JVM,
- "Java Cryptography Extensions are not installed on this JVM."
- +" Maximum supported key length %s - minimum required %d",
- aesLen, minKeyLength);
- }
-
- /**
- * Get the default realm.
- * <p>
- * Not having a default realm may be harmless, so is noted at info.
- * All other invocation failures are downgraded to warn, as
- * follow-on actions may still work.
- * failure to invoke the method via introspection is rejected,
- * as it's a sign of JVM compatibility issues that may have other
- * consequences
- */
- protected void printDefaultRealm() {
- try {
- println("Default Realm = %s",
- getDefaultRealm());
- } catch (ClassNotFoundException
- | IllegalAccessException
- | NoSuchMethodException e) {
-
- throw new KerberosDiagsFailure(CAT_JVM, e,
- "Failed to invoke krb5.Config.getDefaultRealm: %s", e);
- } catch (InvocationTargetException e) {
- Throwable cause = e.getCause() != null ? e.getCause() : e;
- if (cause.toString().contains(NO_DEFAULT_REALM)) {
- // exception raised if there is no default realm. This is not
- // always a problem, so downgrade to a message.
- println("Host has no default realm");
- LOG.debug(cause.toString(), cause);
- } else {
- println("Kerberos.getDefaultRealm() failed: %s\n%s",
- cause,
- org.apache.hadoop.util.StringUtils.stringifyException(cause));
- }
- }
- }
-
- /**
- * Locate the krb5.conf file and dump it.
- * No-op on windows.
- * @throws IOException
- */
- private void validateKrb5File() throws IOException {
- if (!Shell.WINDOWS) {
- title("Locating Kerberos configuration file");
- String krbPath = "/etc/krb5.conf";
- String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF);
- if (jvmKrbPath != null) {
- println("Setting kerberos path from sysprop %s: %s",
- JAVA_SECURITY_KRB5_CONF, jvmKrbPath);
- krbPath = jvmKrbPath;
- }
-
- String krb5name = System.getenv(KRB5_CCNAME);
- if (krb5name != null) {
- println("Setting kerberos path from environment variable %s: %s",
- KRB5_CCNAME, krb5name);
- krbPath = krb5name;
- if (jvmKrbPath != null) {
- println("Warning - both %s and %s were set - %s takes priority",
- JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME);
- }
- }
-
- File krbFile = new File(krbPath);
- println("Kerberos configuration file = %s", krbFile);
- failif(!krbFile.exists(),
- CAT_KERBEROS,
- "Kerberos configuration file %s not found", krbFile);
- dump(krbFile);
- }
- }
-
- /**
- * Dump a keytab: list all principals.
- * @param keytabFile the keytab file
- * @throws IOException IO problems
- */
- public void dumpKeytab(File keytabFile) throws IOException {
- title("Examining keytab %s", keytabFile);
- File kt = keytabFile.getCanonicalFile();
- failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
- failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt);
-
- String[] names = getPrincipalNames(keytabFile.getCanonicalPath(),
- Pattern.compile(".*"));
- println("keytab entry count: %d", names.length);
- for (String name : names) {
- println(" %s", name);
- }
- println("-----");
- }
-
- /**
- * Log in from a keytab, dump the UGI, validate it, then try and log in again.
- * That second-time login catches JVM/Hadoop compatibility problems.
- * @throws IOException
- */
- private void loginFromKeytab() throws IOException {
- UserGroupInformation ugi;
- String identity;
- if (keytab != null) {
- File kt = keytab.getCanonicalFile();
- println("Using keytab %s principal %s", kt, principal);
- identity = principal;
-
- failif(StringUtils.isEmpty(principal), CAT_KERBEROS,
- "No principal defined");
- ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath());
- dumpUGI(identity, ugi);
- validateUGI(principal, ugi);
-
- title("Attempting to log in from keytab again");
- // package scoped -hence the reason why this class must be in the
- // hadoop.security package
- setShouldRenewImmediatelyForTests(true);
- // attempt a new login
- ugi.reloginFromKeytab();
- } else {
- println("No keytab: logging is as current user");
- }
- }
-
- /**
- * Dump a UGI.
- * @param title title of this section
- * @param ugi UGI to dump
- * @throws IOException
- */
- private void dumpUGI(String title, UserGroupInformation ugi)
- throws IOException {
- title(title);
- println("UGI instance = %s", ugi);
- println("Has kerberos credentials: %b", ugi.hasKerberosCredentials());
- println("Authentication method: %s", ugi.getAuthenticationMethod());
- println("Real Authentication method: %s",
- ugi.getRealAuthenticationMethod());
- title("Group names");
- for (String name : ugi.getGroupNames()) {
- println(name);
- }
- title("Credentials");
- Credentials credentials = ugi.getCredentials();
- List<Text> secretKeys = credentials.getAllSecretKeys();
- title("Secret keys");
- if (!secretKeys.isEmpty()) {
- for (Text secret: secretKeys) {
- println("%s", secret);
- }
- } else {
- println("(none)");
- }
-
- dumpTokens(ugi);
- }
-
- /**
- * Validate the UGI: verify it is kerberized.
- * @param messagePrefix message in exceptions
- * @param user user to validate
- */
- private void validateUGI(String messagePrefix, UserGroupInformation user) {
- failif(!user.hasKerberosCredentials(),
- CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user);
- failif(user.getAuthenticationMethod() == null,
- CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user);
- }
-
- /**
- * A cursory look at the {@code kinit} executable.
- * If it is an absolute path: it must exist with a size > 0.
- * If it is just a command, it has to be on the path. There's no check
- * for that -but the PATH is printed out.
- */
- private void validateKinitExecutable() {
- String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, "");
- if (!kinit.isEmpty()) {
- File kinitPath = new File(kinit);
- println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath);
- if (kinitPath.isAbsolute()) {
- failif(!kinitPath.exists(), CAT_KERBEROS,
- "%s executable does not exist: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- failif(!kinitPath.isFile(), CAT_KERBEROS,
- "%s path does not refer to a file: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- failif(kinitPath.length() == 0, CAT_KERBEROS,
- "%s file is empty: %s",
- KERBEROS_KINIT_COMMAND, kinitPath);
- } else {
- println("Executable %s is relative -must be on the PATH", kinit);
- printEnv("PATH");
- }
- }
- }
-
- /**
- * Try to load the SASL resolver.
- * @param saslPropsResolverKey key for the SASL resolver
- */
- private void validateSasl(String saslPropsResolverKey) {
- title("Resolving SASL property %s", saslPropsResolverKey);
- String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey);
- try {
- Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
- saslPropsResolverKey,
- SaslPropertiesResolver.class, SaslPropertiesResolver.class);
- println("Resolver is %s", resolverClass);
- } catch (RuntimeException e) {
- throw new KerberosDiagsFailure(CAT_SASL, e,
- "Failed to load %s class %s",
- saslPropsResolverKey, saslPropsResolver);
- }
- }
-
- /**
- * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE}
- * property.
- */
- private void validateJAAS() {
- String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE);
- if (jaasFilename != null) {
- title("JAAS");
- File jaasFile = new File(jaasFilename);
- println("JAAS file is defined in %s: %s",
- SUN_SECURITY_JAAS_FILE, jaasFile);
- failif(!jaasFile.exists(), CAT_JAAS,
- "JAAS file does not exist: %s", jaasFile);
- failif(!jaasFile.isFile(), CAT_JAAS,
- "Specified JAAS file is not a file: %s", jaasFile);
- }
- }
-
- /**
- * Dump all tokens of a user
- * @param user user
- */
- public void dumpTokens(UserGroupInformation user) {
- Collection<Token<? extends TokenIdentifier>> tokens
- = user.getCredentials().getAllTokens();
- title("Token Count: %d", tokens.size());
- for (Token<? extends TokenIdentifier> token : tokens) {
- println("Token %s", token.getKind());
- }
- }
-
- /**
- * Set the System property to true; return the old value for caching
- * @param sysprop property
- * @return the previous value
- */
- private boolean getAndSet(String sysprop) {
- boolean old = Boolean.getBoolean(sysprop);
- System.setProperty(sysprop, "true");
- return old;
- }
-
- /**
- * Flush all active output channels, including {@Code System.err},
- * so as to stay in sync with any JRE log messages.
- */
- private void flush() {
- if (out != null) {
- out.flush();
- } else {
- System.out.flush();
- }
- System.err.flush();
- }
-
- /**
- * Format and print a line of output.
- * This goes to any output file, or
- * is logged at info. The output is flushed before and after, to
- * try and stay in sync with JRE logging.
- * @param format format string
- * @param args any arguments
- */
- @VisibleForTesting
- public void println(String format, Object... args) {
- println(format(format, args));
- }
-
- /**
- * Print a line of output. This goes to any output file, or
- * is logged at info. The output is flushed before and after, to
- * try and stay in sync with JRE logging.
- * @param msg message string
- */
- @VisibleForTesting
- private void println(String msg) {
- flush();
- if (out != null) {
- out.println(msg);
- } else {
- LOG.info(msg);
- }
- flush();
- }
-
- /**
- * Print a title entry
- * @param format format string
- * @param args any arguments
- */
- private void title(String format, Object... args) {
- println("");
- println("");
- String msg = "== " + format(format, args) + " ==";
- println(msg);
- println("");
- }
-
- /**
- * Print a system property, or {@link #UNSET} if unset.
- * @param property property to print
- */
- private void printSysprop(String property) {
- println("%s = \"%s\"", property,
- System.getProperty(property, UNSET));
- }
-
- /**
- * Print a configuration option, or {@link #UNSET} if unset.
- * @param option option to print
- */
- private void printConfOpt(String option) {
- println("%s = \"%s\"", option, conf.get(option, UNSET));
- }
-
- /**
- * Print an environment variable's name and value; printing
- * {@link #UNSET} if it is not set
- * @param variable environment variable
- */
- private void printEnv(String variable) {
- String env = System.getenv(variable);
- println("%s = \"%s\"", variable, env != null ? env : UNSET);
- }
-
- /**
- * Dump any file to standard out; add a trailing newline
- * @param file file to dump
- * @throws IOException IO problems
- */
- public void dump(File file) throws IOException {
- try (FileInputStream in = new FileInputStream(file)) {
- for (String line : IOUtils.readLines(in)) {
- println("%s", line);
- }
- }
- println("");
- }
-
- /**
- * Format and raise a failure
- *
- * @param category category for exception
- * @param message string formatting message
- * @param args any arguments for the formatting
- * @throws KerberosDiagsFailure containing the formatted text
- */
- private void fail(String category, String message, Object... args)
- throws KerberosDiagsFailure {
- throw new KerberosDiagsFailure(category, message, args);
- }
-
- /**
- * Conditional failure with string formatted arguments
- * @param condition failure condition
- * @param category category for exception
- * @param message string formatting message
- * @param args any arguments for the formatting
- * @throws KerberosDiagsFailure containing the formatted text
- * if the condition was met
- */
- private void failif(boolean condition,
- String category,
- String message,
- Object... args)
- throws KerberosDiagsFailure {
- if (condition) {
- fail(category, message, args);
- }
- }
-
- /**
- * Format a string, treating a call where there are no varags values
- * as a string to pass through unformatted.
- * @param message message, which is either a format string + args, or
- * a general string
- * @param args argument array
- * @return a string for printing.
- */
- public static String format(String message, Object... args) {
- if (args.length == 0) {
- return message;
- } else {
- return String.format(message, args);
- }
- }
-
- /**
- * Diagnostics failures return the exit code 41, "unauthorized".
- *
- * They have a category, initially for testing: the category can be
- * validated without having to match on the entire string.
- */
- public static class KerberosDiagsFailure extends ExitUtil.ExitException {
- private final String category;
-
- public KerberosDiagsFailure(String category, String message) {
- super(41, category + ": " + message);
- this.category = category;
- }
-
- public KerberosDiagsFailure(String category, String message, Object... args) {
- this(category, format(message, args));
- }
-
- public KerberosDiagsFailure(String category, Throwable throwable,
- String message, Object... args) {
- this(category, message, args);
- initCause(throwable);
- }
-
- public String getCategory() {
- return category;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
deleted file mode 100644
index 6efa880..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import java.util.regex.Pattern;
-
-/**
- * Utility class to validate strings against a predefined pattern.
- */
-public class PatternValidator {
-
- public static final String E_INVALID_NAME =
- "Invalid name %s does not match the pattern pattern %s ";
- private final Pattern valid;
- private final String pattern;
-
- public PatternValidator(String pattern) {
- this.pattern = pattern;
- valid = Pattern.compile(pattern);
- }
-
- /**
- * Validate the name -restricting it to the set defined in
- * @param name name to validate
- * @throws IllegalArgumentException if not a valid name
- */
- public void validate(String name) {
- if (!matches(name)) {
- throw new IllegalArgumentException(
- String.format(E_INVALID_NAME, name, pattern));
- }
- }
-
- /**
- * Query to see if the pattern matches
- * @param name name to validate
- * @return true if the string matches the pattern
- */
- public boolean matches(String name) {
- return valid.matcher(name).matches();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
deleted file mode 100644
index 2dbf37f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * a scanner which can take an input string for a range or scan the lot.
- */
-public class PortScanner {
- private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
- private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$");
-
- private List<Integer> remainingPortsToCheck;
-
- public PortScanner() {
- }
-
- public void setPortRange(String input) throws BadConfigException {
- // first split based on commas
- Set<Integer> inputPorts= new TreeSet<Integer>();
- String[] ranges = input.split(",");
- for ( String range : ranges ) {
- if (range.trim().isEmpty()) {
- continue;
- }
- Matcher m = SINGLE_NUMBER.matcher(range.trim());
- if (m.find()) {
- inputPorts.add(Integer.parseInt(m.group()));
- continue;
- }
- m = NUMBER_RANGE.matcher(range.trim());
- if (m.find()) {
- String[] boundaryValues = m.group(0).split("-");
- int start = Integer.parseInt(boundaryValues[0].trim());
- int end = Integer.parseInt(boundaryValues[1].trim());
- if (end < start) {
- throw new BadConfigException("End of port range is before start: "
- + range + " in input: " + input);
- }
- for (int i = start; i < end + 1; i++) {
- inputPorts.add(i);
- }
- continue;
- }
- throw new BadConfigException("Bad port range: " + range + " in input: "
- + input);
- }
- if (inputPorts.size() == 0) {
- throw new BadConfigException("No ports found in range: " + input);
- }
- this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
- }
-
- public List<Integer> getRemainingPortsToCheck() {
- return remainingPortsToCheck;
- }
-
- public int getAvailablePort() throws SliderException, IOException {
- if (remainingPortsToCheck != null) {
- return getAvailablePortViaPortArray();
- } else {
- return SliderUtils.getOpenPort();
- }
- }
-
- private int getAvailablePortViaPortArray() throws SliderException {
- boolean found = false;
- int availablePort = -1;
- Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
- while (portsToCheck.hasNext() && !found) {
- int portToCheck = portsToCheck.next();
- found = SliderUtils.isPortAvailable(portToCheck);
- if (found) {
- availablePort = portToCheck;
- portsToCheck.remove();
- }
- }
-
- if (availablePort < 0) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
- "No available ports found in configured range {}",
- remainingPortsToCheck);
- }
-
- return availablePort;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
deleted file mode 100644
index 9d00b3c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.SerializationConfig;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * JSON-serializable description of a published key-val configuration.
- *
- * The values themselves are not serialized in the external view; they have
- * to be served up by the far end
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class PublishedConfiguration {
-
- public String description;
- public long updated;
-
- public String updatedTime;
-
- public Map<String, String> entries = new HashMap<>();
-
- public PublishedConfiguration() {
- }
-
- /**
- * build an empty published configuration
- * @param description configuration description
- */
- public PublishedConfiguration(String description) {
- this.description = description;
- }
-
- /**
- * Build a configuration from the entries
- * @param description configuration description
- * @param entries entries to put
- */
- public PublishedConfiguration(String description,
- Iterable<Map.Entry<String, String>> entries) {
- this.description = description;
- putValues(entries);
- }
-
- /**
- * Build a published configuration, using the keys from keysource,
- * but resolving the values from the value source, via Configuration.get()
- * @param description configuration description
- * @param keysource source of keys
- * @param valuesource source of values
- */
- public PublishedConfiguration(String description,
- Iterable<Map.Entry<String, String>> keysource,
- Configuration valuesource) {
- this.description = description;
- putValues(ConfigHelper.resolveConfiguration(keysource, valuesource));
- }
-
-
- /**
- * Is the configuration empty. This means either that it has not
- * been given any values, or it is stripped down copy set down over the
- * wire.
- * @return true if it is empty
- */
- public boolean isEmpty() {
- return entries.isEmpty();
- }
-
-
- public void setUpdated(long updated) {
- this.updated = updated;
- this.updatedTime = new Date(updated).toString();
- }
-
- public long getUpdated() {
- return updated;
- }
-
- /**
- * Set the values from an iterable (this includes a Hadoop Configuration
- * and Java properties object).
- * Any existing value set is discarded
- * @param entries entries to put
- */
- public void putValues(Iterable<Map.Entry<String, String>> entries) {
- this.entries = new HashMap<String, String>();
- for (Map.Entry<String, String> entry : entries) {
- this.entries.put(entry.getKey(), entry.getValue());
- }
-
- }
-
- /**
- * Convert to Hadoop XML
- * @return the configuration as a Hadoop Configuratin
- */
- public Configuration asConfiguration() {
- Configuration conf = new Configuration(false);
- try {
- ConfigHelper.addConfigMap(conf, entries, "");
- } catch (BadConfigException e) {
- // triggered on a null value; switch to a runtime (and discard the stack)
- throw new RuntimeException(e.toString());
- }
- return conf;
- }
-
- public String asConfigurationXML() throws IOException {
- return ConfigHelper.toXml(asConfiguration());
- }
-
- /**
- * Convert values to properties
- * @return a property file
- */
- public Properties asProperties() {
- Properties props = new Properties();
- props.putAll(entries);
- return props;
- }
-
- /**
- * Return the values as json string
- * @return the JSON representation
- * @throws IOException marshalling failure
- */
- public String asJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
- String json = mapper.writeValueAsString(entries);
- return json;
- }
-
-
- /**
- * This makes a copy without the nested content -so is suitable
- * for returning as part of the list of a parent's values
- * @return the copy
- */
- public PublishedConfiguration shallowCopy() {
- PublishedConfiguration that = new PublishedConfiguration();
- that.description = this.description;
- that.updated = this.updated;
- that.updatedTime = this.updatedTime;
- return that;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("PublishedConfiguration{");
- sb.append("description='").append(description).append('\'');
- sb.append(" entries = ").append(entries.size());
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Create an outputter for a given format
- * @param format format to use
- * @return an instance of output
- */
- public PublishedConfigurationOutputter createOutputter(ConfigFormat format) {
- return PublishedConfigurationOutputter.createOutputter(format, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
deleted file mode 100644
index 88ecf2c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-import org.yaml.snakeyaml.DumperOptions;
-import org.yaml.snakeyaml.DumperOptions.FlowStyle;
-import org.yaml.snakeyaml.Yaml;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.StringWriter;
-import java.util.Properties;
-
-/**
- * Output a published configuration
- */
-public abstract class PublishedConfigurationOutputter {
-
- private static final String COMMENTS = "Generated by Apache Slider";
-
- protected final PublishedConfiguration owner;
-
- protected PublishedConfigurationOutputter(PublishedConfiguration owner) {
- this.owner = owner;
- }
-
- /**
- * Save the config to a destination file, in the format of this outputter
- * @param dest destination file
- * @throws IOException
- */
-/* JDK7
- public void save(File dest) throws IOException {
- try(FileOutputStream out = new FileOutputStream(dest)) {
- save(out);
- out.close();
- }
- }
-*/
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
- }
-
- /**
- * Save the content. The default saves the asString() value
- * to the output stream
- * @param out output stream
- * @throws IOException
- */
- public void save(OutputStream out) throws IOException {
- IOUtils.write(asString(), out, Charsets.UTF_8);
- }
- /**
- * Convert to a string
- * @return the string form
- * @throws IOException
- */
- public abstract String asString() throws IOException;
-
- /**
- * Create an outputter for the chosen format
- * @param format format enumeration
- * @param owner owning config
- * @return the outputter
- */
-
- public static PublishedConfigurationOutputter createOutputter(ConfigFormat format,
- PublishedConfiguration owner) {
- Preconditions.checkNotNull(owner);
- switch (format) {
- case XML:
- case HADOOP_XML:
- return new XmlOutputter(owner);
- case PROPERTIES:
- return new PropertiesOutputter(owner);
- case JSON:
- return new JsonOutputter(owner);
- case ENV:
- return new EnvOutputter(owner);
- case TEMPLATE:
- return new TemplateOutputter(owner);
- case YAML:
- return new YamlOutputter(owner);
- default:
- throw new RuntimeException("Unsupported format :" + format);
- }
- }
-
- public static class XmlOutputter extends PublishedConfigurationOutputter {
-
-
- private final Configuration configuration;
-
- public XmlOutputter(PublishedConfiguration owner) {
- super(owner);
- configuration = owner.asConfiguration();
- }
-
- @Override
- public void save(OutputStream out) throws IOException {
- configuration.writeXml(out);
- }
-
- @Override
- public String asString() throws IOException {
- return ConfigHelper.toXml(configuration);
- }
-
- public Configuration getConfiguration() {
- return configuration;
- }
- }
-
- public static class PropertiesOutputter extends PublishedConfigurationOutputter {
-
- private final Properties properties;
-
- public PropertiesOutputter(PublishedConfiguration owner) {
- super(owner);
- properties = owner.asProperties();
- }
-
- @Override
- public void save(OutputStream out) throws IOException {
- properties.store(out, COMMENTS);
- }
-
-
- public String asString() throws IOException {
- StringWriter sw = new StringWriter();
- properties.store(sw, COMMENTS);
- return sw.toString();
- }
- }
-
-
- public static class JsonOutputter extends PublishedConfigurationOutputter {
-
- public JsonOutputter(PublishedConfiguration owner) {
- super(owner);
- }
-
- @Override
- public String asString() throws IOException {
- return owner.asJson();
- }
- }
-
-
- public static class EnvOutputter extends PublishedConfigurationOutputter {
-
- public EnvOutputter(PublishedConfiguration owner) {
- super(owner);
- }
-
- @Override
- public String asString() throws IOException {
- if (!owner.entries.containsKey("content")) {
- throw new IOException("Configuration has no content field and cannot " +
- "be retrieved as type 'env'");
- }
- String content = owner.entries.get("content");
- return ConfigUtils.replaceProps(owner.entries, content);
- }
- }
-
- public static class TemplateOutputter extends EnvOutputter {
- public TemplateOutputter(PublishedConfiguration owner) {
- super(owner);
- }
- }
-
- public static class YamlOutputter extends PublishedConfigurationOutputter {
-
- private final Yaml yaml;
-
- public YamlOutputter(PublishedConfiguration owner) {
- super(owner);
- DumperOptions options = new DumperOptions();
- options.setDefaultFlowStyle(FlowStyle.BLOCK);
- yaml = new Yaml(options);
- }
-
- public String asString() throws IOException {
- return yaml.dump(owner.entries);
- }
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[51/58] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd
and merge service cmd into application cmd. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
deleted file mode 100644
index 65f0472..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-
-/**
- * Resolve registry entries
- *
- * --path {path}
- * --out {destfile}
- * --verbose
- * --list
- */
-@Parameters(commandNames = {ACTION_RESOLVE},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-public class ActionResolveArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_RESOLVE
- + " "
- + ARG_PATH + " <path> "
- + "[" + ARG_LIST + "] "
- + "[" + ARG_OUTPUT + " <filename> ] "
- + "[" + ARG_DESTDIR + " <directory> ] "
- ;
- public ActionResolveArgs() {
- }
-
- @Override
- public String getActionName() {
- return ACTION_RESOLVE;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_PATH},
- description = "resolve a path")
- public String path;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "destination directory for operations")
- public File destdir;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "dest file")
- public File out;
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_RESOLVE).append(" ");
- sb.append(ARG_PATH).append(" ").append(path).append(" ");
- if (list) {
- sb.append(ARG_LIST).append(" ");
- }
- if (destdir != null) {
- sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
- }
- if (out != null) {
- sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
- }
- return sb.toString();
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (StringUtils.isEmpty(path)) {
- throw new BadCommandArgumentsException("Missing mandatory argument "
- + ARG_PATH);
- }
- if (list && out != null) {
- throw new BadCommandArgumentsException("Argument "
- + ARG_OUTPUT +
- " not supported for " + ARG_LIST);
- }
- if (out != null && destdir != null) {
- throw new BadCommandArgumentsException(
- ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
- );
- }
- }
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public boolean isList() {
- return list;
- }
-
- public void setList(boolean list) {
- this.list = list;
- }
-
- public File getDestdir() {
- return destdir;
- }
-
- public void setDestdir(File destdir) {
- this.destdir = destdir;
- }
-
- public File getOut() {
- return out;
- }
-
- public void setOut(File out) {
- this.out = out;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
deleted file mode 100644
index b03dc92..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_RESOURCE},
- commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
-
-public class ActionResourceArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_RESOURCE;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install the resource(s)")
- public boolean install;
-
- @Parameter(names = {ARG_DELETE},
- description = "Delete the file")
- public boolean delete;
-
- @Parameter(names = {ARG_LIST},
- description = "List of installed files")
- public boolean list;
-
- @Parameter(names = {ARG_RESOURCE},
- description = "Name of the file or directory")
- public String resource;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "The name of the folder in which to store the resources")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
deleted file mode 100644
index 31f25ef..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_STATUS},
- commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
-
-public class ActionStatusArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STATUS;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for the status information")
- public String output;
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Lifetime of the service from the time of request")
- public boolean lifetime;
-
- public String getOutput() {
- return output;
- }
-
- public void setOutput(String output) {
- this.output = output;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
deleted file mode 100644
index 175e367..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_START },
- commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
-public class ActionThawArgs extends AbstractActionArgs implements
- WaitTimeAccessor,
- LaunchArgsAccessor {
-
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_START;
- }
-
- @Override
- public int getWaittime() {
- return launchArgs.getWaittime();
- }
-
- @ParametersDelegate
- LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Life time of the service since service started at"
- + " running state")
- public long lifetime;
-
- @Override
- public String getRmAddress() {
- return launchArgs.getRmAddress();
- }
-
- @Override
- public void setWaittime(int waittime) {
- launchArgs.setWaittime(waittime);
- }
-
-
- @Override
- public File getOutputFile() {
- return launchArgs.getOutputFile();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
deleted file mode 100644
index cf48513..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_TOKENS},
- commandDescription = "save tokens to a file or list tokens in a file")
-public class ActionTokensArgs extends AbstractActionArgs {
-
- public static final String DUPLICATE_ARGS = "Only one of " +
- ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
-
- public static final String MISSING_KT_PROVIDER =
- "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
- + " must be provided";
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_TOKENS;
- }
-
- @Parameter(names = {ARG_OUTPUT},
- description = "File to write")
- public File output;
-
- @Parameter(names = {ARG_SOURCE},
- description = "source file")
- public File source;
-
- @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
- public File keytab;
-
- @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
- public String principal="";
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (output != null && source != null) {
- throw new BadCommandArgumentsException(DUPLICATE_ARGS);
- }
-
- // this is actually a !xor
- if (keytab != null ^ !principal.isEmpty()) {
- throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
deleted file mode 100644
index 00af69a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
-
-public class ActionUpdateArgs extends AbstractActionArgs {
-
- @Parameter(names = {
- ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
- public long lifetime;
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_UPDATE;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
deleted file mode 100644
index 00151f4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Static argument manipulation operations
- */
-public class ArgOps {
-
- private static final Logger
- log = LoggerFactory.getLogger(ArgOps.class);
-
- /**
- * create a 3-tuple
- */
- public static List<Object> triple(String msg, int min, int max) {
- List<Object> l = new ArrayList<>(3);
- l.add(msg);
- l.add(min);
- l.add(max);
- return l;
- }
-
- public static void applyFileSystemBinding(String filesystemBinding,
- Configuration conf) {
- if (filesystemBinding != null) {
- //filesystem argument was set -this overwrites any defaults in the
- //configuration
- FileSystem.setDefaultUri(conf, filesystemBinding);
- }
- }
-
- public static void splitPairs(Collection<String> pairs,
- Map<String, String> dest) {
- for (String prop : pairs) {
- String[] keyval = prop.split("=", 2);
- if (keyval.length == 2) {
- dest.put(keyval[0], keyval[1]);
- }
- }
- }
-
-
- public static void applyDefinitions(Map<String, String> definitionMap,
- Configuration conf) {
- for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- log.debug("configuration[{}]<=\"{}\"", key, val);
- conf.set(key, val, "command line");
- }
- }
-
- /**
- * Create a map from a tuple list like ['worker','2','master','1] into a map
- * ['worker':'2',"master":'1'];
- * Duplicate entries also trigger errors
- * @param description description for errors
- * @param list list to conver to tuples
- * @return the map of key value pairs -unordered.
- * @throws BadCommandArgumentsException odd #of arguments received
- */
- public static Map<String, String> convertTupleListToMap(String description,
- List<String> list) throws
- BadCommandArgumentsException {
- Map<String, String> results = new HashMap<>();
- if (list != null && !list.isEmpty()) {
- int size = list.size();
- if (size % 2 != 0) {
- //odd number of elements, not permitted
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_PARSE_FAILURE + description);
- }
- for (int count = 0; count < size; count += 2) {
- String key = list.get(count);
- String val = list.get(count + 1);
- if (results.get(key) != null) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_DUPLICATE_ENTRY + description
- + ": " + key);
- }
- results.put(key, val);
- }
- }
- return results;
- }
-
- /**
- * Create a map from a tuple list like
- * ['worker','heapsize','5G','master','heapsize','2M'] into a map
- * ['worker':'2',"master":'1'];
- * Duplicate entries also trigger errors
-
- * @throws BadCommandArgumentsException odd #of arguments received
- */
- public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
- List<String> list) throws BadCommandArgumentsException {
-
- Map<String, Map<String, String>> results = new HashMap<>();
- if (list != null && !list.isEmpty()) {
- int size = list.size();
- if (size % 3 != 0) {
- //wrong number of elements, not permitted
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_PARSE_FAILURE + description);
- }
- for (int count = 0; count < size; count += 3) {
- String role = list.get(count);
- String key = list.get(count + 1);
- String val = list.get(count + 2);
- Map<String, String> roleMap = results.get(role);
- if (roleMap == null) {
- //demand create new role map
- roleMap = new HashMap<>();
- results.put(role, roleMap);
- }
- if (roleMap.get(key) != null) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_DUPLICATE_ENTRY + description
- + ": for key " + key + " under " + role);
- }
- roleMap.put(key, val);
- }
- }
- return results;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
deleted file mode 100644
index 67571e2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Here are all the arguments that may be parsed by the client or server
- * command lines.
- *
- * Important: Please keep the main list in alphabetical order
- * so it is easier to see what arguments are there
- */
-public interface Arguments {
-
- String ARG_FILE = "--file";
- String ARG_FILE_SHORT = "-f";
- String ARG_BASE_PATH = "--basepath";
- String ARG_COMPONENT = "--component";
- String ARG_COMPONENT_SHORT = "--comp";
- String ARG_COMPONENTS = "--components";
- String ARG_COMP_OPT= "--compopt";
- String ARG_COMP_OPT_SHORT = "--co";
- String ARG_CONFIG = "--config";
- String ARG_CONTAINERS = "--containers";
- String ARG_DEBUG = "--debug";
- String ARG_DEFINE = "-D";
- String ARG_DELETE = "--delete";
- String ARG_DEST = "--dest";
- String ARG_DESTDIR = "--destdir";
- String ARG_EXAMPLE = "--example";
- String ARG_EXAMPLE_SHORT = "-e";
- String ARG_FOLDER = "--folder";
- String ARG_FORCE = "--force";
- String ARG_FORMAT = "--format";
- String ARG_GETCONF = "--getconf";
- String ARG_GETEXP = "--getexp";
- String ARG_GETFILES = "--getfiles";
- String ARG_HELP = "--help";
- String ARG_IMAGE = "--image";
- String ARG_INSTALL = "--install";
- String ARG_INTERNAL = "--internal";
- String ARG_KEYLEN = "--keylen";
- String ARG_KEYTAB = "--keytab";
- String ARG_KEYTABINSTALL = ARG_INSTALL;
- String ARG_KEYTABDELETE = ARG_DELETE;
- String ARG_KEYTABLIST = "--list";
- String ARG_LIST = "--list";
- String ARG_LISTCONF = "--listconf";
- String ARG_LISTEXP = "--listexp";
- String ARG_LISTFILES = "--listfiles";
- String ARG_LIVE = "--live";
- String ARG_MANAGER = "--manager";
- String ARG_MANAGER_SHORT = "--m";
- String ARG_MESSAGE = "--message";
- String ARG_NAME = "--name";
- String ARG_OPTION = "--option";
- String ARG_OPTION_SHORT = "-O";
- String ARG_OUTPUT = "--out";
- String ARG_OUTPUT_SHORT = "-o";
- String ARG_OVERWRITE = "--overwrite";
- String ARG_PACKAGE = "--package";
- String ARG_PATH = "--path";
- String ARG_PRINCIPAL = "--principal";
- String ARG_QUEUE = "--queue";
- String ARG_SHORT_QUEUE = "-q";
- String ARG_LIFETIME = "--lifetime";
- String ARG_RESOURCE = "--resource";
- String ARG_RESOURCE_MANAGER = "--rm";
- String ARG_SECURE = "--secure";
- String ARG_SERVICETYPE = "--servicetype";
- String ARG_SERVICES = "--services";
- String ARG_SOURCE = "--source";
- String ARG_STATE = "--state";
- String ARG_SYSPROP = "-S";
- String ARG_USER = "--user";
- String ARG_UPLOAD = "--upload";
- String ARG_VERBOSE = "--verbose";
- String ARG_VERSION = "--version";
- String ARG_WAIT = "--wait";
-/*
- STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
- RIGHT PLACE IN THE LIST
- */
-
- // Tha path in hdfs to be read by Service AM
- String ARG_SERVICE_DEF_PATH = "-cluster-uri";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
deleted file mode 100644
index 84d3bbd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-
-import java.util.Collection;
-
-/**
- * Client CLI Args
- */
-
-public class ClientArgs extends CommonArgs {
-
- // =========================================================
- // Keep all of these in alphabetical order. Thanks.
- // =========================================================
-
- private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
- private final ActionClientArgs actionClientArgs = new ActionClientArgs();
- private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
- private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
- private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
- private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
- private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
- private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
- private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
- private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
- private final ActionListArgs actionListArgs = new ActionListArgs();
- private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
- private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
- private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
- private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
- private final ActionThawArgs actionThawArgs = new ActionThawArgs();
- private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
- private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
-
- public ClientArgs(String[] args) {
- super(args);
- }
-
- public ClientArgs(Collection args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
-
- addActions(
- actionBuildArgs,
- actionCreateArgs,
- actionDependencyArgs,
- actionDestroyArgs,
- actionFlexArgs,
- actionFreezeArgs,
- actionHelpArgs,
- actionStatusArgs,
- actionThawArgs
- );
- }
-
- @Override
- public void applyDefinitions(Configuration conf) throws
- BadCommandArgumentsException {
- super.applyDefinitions(conf);
- }
-
-
- public ActionBuildArgs getActionBuildArgs() {
- return actionBuildArgs;
- }
-
- public ActionUpdateArgs getActionUpdateArgs() {
- return actionUpdateArgs;
- }
-
- public ActionCreateArgs getActionCreateArgs() {
- return actionCreateArgs;
- }
-
- public ActionDependencyArgs getActionDependencyArgs() {
- return actionDependencyArgs;
- }
-
- public ActionFlexArgs getActionFlexArgs() {
- return actionFlexArgs;
- }
-
- /**
- * Look at the chosen action and bind it as the core action for the operation.
- * @throws SliderException bad argument or similar
- */
- @Override
- public void applyAction() throws SliderException {
- String action = getAction();
- if (SliderUtils.isUnset(action)) {
- action = ACTION_HELP;
- }
- switch (action) {
- case ACTION_BUILD:
- bindCoreAction(actionBuildArgs);
- break;
-
- case ACTION_CREATE:
- bindCoreAction(actionCreateArgs);
- break;
-
- case ACTION_STOP:
- bindCoreAction(actionFreezeArgs);
- break;
-
- case ACTION_START:
- bindCoreAction(actionThawArgs);
- break;
-
- case ACTION_DEPENDENCY:
- bindCoreAction(actionDependencyArgs);
- break;
-
- case ACTION_DESTROY:
- bindCoreAction(actionDestroyArgs);
- break;
-
- case ACTION_EXISTS:
- bindCoreAction(actionExistsArgs);
- break;
-
- case ACTION_FLEX:
- bindCoreAction(actionFlexArgs);
- break;
-
- case ACTION_HELP:
- bindCoreAction(actionHelpArgs);
- break;
-
- case ACTION_KEYTAB:
- bindCoreAction(actionKeytabArgs);
- break;
-
- case ACTION_LIST:
- bindCoreAction(actionListArgs);
- break;
-
- case ACTION_REGISTRY:
- bindCoreAction(actionRegistryArgs);
- break;
-
- case ACTION_RESOLVE:
- bindCoreAction(actionResolveArgs);
- break;
-
- case ACTION_RESOURCE:
- bindCoreAction(actionResourceArgs);
- break;
-
- case ACTION_STATUS:
- bindCoreAction(actionStatusArgs);
- break;
-
- case ACTION_TOKENS:
- bindCoreAction(actionTokenArgs);
- break;
-
- case ACTION_UPDATE:
- bindCoreAction(actionUpdateArgs);
- break;
- default:
- throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
- + " " + action);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
deleted file mode 100644
index 145b44a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterDescription;
-import com.beust.jcommander.ParameterException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This class contains the common argument set for all tne entry points,
- * and the core parsing logic to verify that the action is on the list
- * of allowed actions -and that the remaining number of arguments is
- * in the range allowed
- */
-
-public abstract class CommonArgs extends ArgOps implements SliderActions,
- Arguments {
-
- protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
-
-
- private static final int DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME = 30;
-
-
- @Parameter(names = ARG_HELP, help = true)
- public boolean help;
-
-
- /**
- -D name=value
-
- Define an HBase configuration option which overrides any options in
- the configuration XML files of the image or in the image configuration
- directory. The values will be persisted.
- Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
-
- */
-
- public Map<String, String> definitionMap = new HashMap<String, String>();
- /**
- * System properties
- */
- public Map<String, String> syspropsMap = new HashMap<String, String>();
-
-
- /**
- * fields
- */
- public final JCommander commander;
- private final String[] args;
-
- private AbstractActionArgs coreAction;
-
- /**
- * get the name: relies on arg 1 being the cluster name in all operations
- * @return the name argument, null if there is none
- */
- public String getClusterName() {
- return coreAction.getServiceName();
- }
-
- protected CommonArgs(String[] args) {
- this.args = args;
- commander = new JCommander(this);
- }
-
- protected CommonArgs(Collection args) {
- List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
- this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
- commander = new JCommander(this);
- }
-
- public String usage() {
- return usage(this, null);
- }
-
- public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
- String result = null;
- StringBuilder helperMessage = new StringBuilder();
- if (commandOfInterest == null) {
- // JCommander.usage is too verbose for a command with many options like
- // slider no short version of that is found Instead, we compose our msg by
- helperMessage.append("\nUsage: service COMMAND [options]\n");
- helperMessage.append("where COMMAND is one of\n");
- for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
- helperMessage.append(String.format("\t%-"
- + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", jcommand,
- serviceArgs.commander.getCommandDescription(jcommand) + "\n"));
- }
- helperMessage
- .append("Most commands print help when invoked without parameters or with --help");
- result = helperMessage.toString();
- } else {
- helperMessage.append("\nUsage: service ").append(commandOfInterest);
- helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <service>" : "");
- helperMessage.append("\n");
- for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
- .get(commandOfInterest).getParameters()) {
- String optional = paramDesc.getParameter().required() ? " (required)"
- : " (optional)";
- String paramName = paramDesc.getParameterized().getType() == Boolean.TYPE ? paramDesc
- .getLongestName() : paramDesc.getLongestName() + " <"
- + paramDesc.getParameterized().getName() + ">";
- helperMessage.append(String.format("\t%-"
- + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", paramName,
- paramDesc.getDescription() + optional + "\n"));
- result = helperMessage.toString();
- }
- }
- return result;
- }
-
- /**
- * Parse routine -includes registering the action-specific argument classes
- * and postprocess it
- * @throws SliderException on any problem
- */
- public void parse() throws SliderException {
- addActionArguments();
- try {
- commander.parse(args);
- } catch (ParameterException e) {
- commander.usage(args[0]);
- throw e;
- }
- //now copy back to this class some of the attributes that are common to all
- //actions
- postProcess();
- }
-
-
- protected void addActions(Object... actions) {
- for (Object action : actions) {
- commander.addCommand(action);
- }
- }
-
- /**
- * Override point to add a set of actions
- */
- protected void addActionArguments() {
-
- }
-
- /**
- * validate args via {@link #validate()}
- * then postprocess the arguments
- */
- public void postProcess() throws SliderException {
- applyAction();
- validate();
-
- //apply entry set
- for (Map.Entry<String, String> entry : syspropsMap.entrySet()) {
- System.setProperty(entry.getKey(), entry.getValue());
- }
- }
-
-
- /**
- * Implementors must implement their action apply routine here
- */
- public abstract void applyAction() throws SliderException;
-
-
- /**
- * Bind the core action; this extracts any attributes that are used
- * across routines
- * @param action action to bind
- */
- protected void bindCoreAction(AbstractActionArgs action) {
- coreAction = action;
-
- splitPairs(coreAction.definitions, definitionMap);
- splitPairs(coreAction.sysprops, syspropsMap);
- }
-
- /**
- * Validate the arguments against the action requested
- */
- public void validate() throws BadCommandArgumentsException, UsageException {
- if (coreAction == null) {
- throw new UsageException(ErrorStrings.ERROR_NO_ACTION + usage());
- }
- log.debug("action={}", getAction());
- // let the action validate itself
- try {
- coreAction.validate();
- } catch (BadCommandArgumentsException e) {
- String badArgMsgBuilder =
- e.getMessage() + System.lineSeparator() + usage(this,
- coreAction.getActionName());
- throw new BadCommandArgumentsException(badArgMsgBuilder);
- }
- }
-
- /**
- * Apply all the definitions on the command line to the configuration
- * @param conf config
- */
- public void applyDefinitions(Configuration conf) throws
- BadCommandArgumentsException {
- applyDefinitions(definitionMap, conf);
- }
-
- public boolean isDebug() {
- return coreAction.debug;
- }
-
-
- public String getAction() {
- return commander.getParsedCommand();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
deleted file mode 100644
index b6cd0a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class ComponentArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * This is a listing of the roles to create
- */
- @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT},
- arity = 2,
- description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
- splitter = DontSplitArguments.class)
- public List<String> componentTuples = new ArrayList<>(0);
-
-
- /**
- * Get the role mapping (may be empty, but never null)
- * @return role mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
- return convertTupleListToMap("component", componentTuples);
- }
-
- public List<String> getComponentTuples() {
- return componentTuples;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
deleted file mode 100644
index 85de615..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.converters.IParameterSplitter;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DontSplitArguments implements IParameterSplitter {
-
- @Override
- public List<String> split(String value) {
- List<String> list = new ArrayList<>(1);
- list.add(value);
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
deleted file mode 100644
index bf194b6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import java.io.File;
-
-/**
- * Launch args for create and start and anything else that can start something
- */
-public interface LaunchArgsAccessor extends WaitTimeAccessor {
- String getRmAddress();
-
- File getOutputFile();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
deleted file mode 100644
index d42510c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-import java.io.File;
-
-/**
- * Any launch-time args
- */
-public class LaunchArgsDelegate extends WaitArgsDelegate implements
- LaunchArgsAccessor {
-
-
- //TODO: do we need this?
- @Parameter(names = ARG_RESOURCE_MANAGER,
- description = "Resource manager hostname:port ",
- required = false)
- private String rmAddress;
-
- @Override
- public String getRmAddress() {
- return rmAddress;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any service report")
- public File outputFile;
-
- @Override
- public File getOutputFile() {
- return outputFile;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
deleted file mode 100644
index 7972716..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Delegate for application and resource options.
- */
-public class OptionArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * Options key value.
- */
- @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
- description = ARG_OPTION + "<name> <value>",
- splitter = DontSplitArguments.class)
- public List<String> optionTuples = new ArrayList<>(0);
-
-
- /**
- * All the app component option triples.
- */
- @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
- description = "Component option " + ARG_COMP_OPT +
- " <component> <name> <option>",
- splitter = DontSplitArguments.class)
- public List<String> compOptTriples = new ArrayList<>(0);
-
- public Map<String, String> getOptionsMap() throws
- BadCommandArgumentsException {
- return convertTupleListToMap(ARG_OPTION, optionTuples);
- }
-
- /**
- * Get the role heap mapping (may be empty, but never null).
- * @return role heap mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, Map<String, String>> getCompOptionMap()
- throws BadCommandArgumentsException {
- return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
deleted file mode 100644
index 040ac64..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.converters.BaseConverter;
-import org.apache.hadoop.fs.Path;
-
-public class PathArgumentConverter extends BaseConverter<Path> {
-
- public PathArgumentConverter(String optionName) {
- super(optionName);
- }
-
- @Override
- public Path convert(String value) {
- return new Path(value);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
deleted file mode 100644
index 800bb24..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Parameters sent by the Client to the AM
- */
-public class ServiceAMArgs extends CommonArgs {
-
- ServiceAMCreateAction createAction = new ServiceAMCreateAction();
-
- public ServiceAMArgs(String[] args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
- addActions(createAction);
- }
-
- // This is the path in hdfs to the service definition JSON file
- public String getServiceDefPath() {
- return createAction.serviceDefPath;
- }
-
- /**
- * Am binding is simple: there is only one action
- */
- @Override
- public void applyAction() {
- bindCoreAction(createAction);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
deleted file mode 100644
index 384e8ba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class ServiceAMCreateAction extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-
- @Parameter(names = ARG_SERVICE_DEF_PATH,
- description = "Path to the service definition JSON file", required = true)
- public String serviceDefPath;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
deleted file mode 100644
index 680bc0f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Actions by client
- */
-public interface SliderActions {
- String ACTION_BUILD = "build";
- String ACTION_CLIENT = "client";
- String ACTION_CREATE = "create";
- String ACTION_DEPENDENCY = "dependency";
- String ACTION_UPDATE = "update";
- String ACTION_UPGRADE = "upgrade";
- String ACTION_DESTROY = "destroy";
- String ACTION_EXISTS = "exists";
- String ACTION_FLEX = "flex";
- String ACTION_STOP = "stop";
- String ACTION_HELP = "help";
- String ACTION_INSTALL_KEYTAB = "install-keytab";
- String ACTION_KEYTAB = "keytab";
- String ACTION_LIST = "list";
-
- String ACTION_REGISTRY = "registry";
- String ACTION_RESOLVE = "resolve";
- String ACTION_RESOURCE = "resource";
- String ACTION_STATUS = "status";
- String ACTION_START = "start";
- String ACTION_TOKENS = "tokens";
-
- String DESCRIBE_ACTION_BUILD =
- "Build a service specification, but do not start it";
- String DESCRIBE_ACTION_CREATE =
- "Create a service, it's equivalent to first invoke build and then start";
- String DESCRIBE_ACTION_DEPENDENCY =
- "Yarn service framework dependency (libraries) management";
- String DESCRIBE_ACTION_UPDATE =
- "Update template for service";
- String DESCRIBE_ACTION_UPGRADE =
- "Rolling upgrade/downgrade the component/containerto a newer/previous version";
- String DESCRIBE_ACTION_DESTROY =
- "Destroy a stopped service, service must be stopped first before destroying.";
- String DESCRIBE_ACTION_EXISTS =
- "Probe for a service running";
- String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
- String DESCRIBE_ACTION_FREEZE =
- "Stop a running service";
- String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
- String DESCRIBE_ACTION_HELP = "Print help information";
- String DESCRIBE_ACTION_LIST =
- "List running services";
- String DESCRIBE_ACTION_REGISTRY =
- "Query the registry of a service";
- String DESCRIBE_ACTION_STATUS =
- "Get the status of a service";
- String DESCRIBE_ACTION_THAW =
- "Start a service with pre-built specification or a previously stopped service";
- String DESCRIBE_ACTION_CLIENT = "Install the service client in the specified directory or obtain a client keystore or truststore";
- String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
- String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
-
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
deleted file mode 100644
index 86f3709..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-public class WaitArgsDelegate extends AbstractArgsDelegate implements
- WaitTimeAccessor {
-
-
- //--wait [timeout]
- @Parameter(names = {ARG_WAIT},
- description = "time to wait for an action to complete")
- public int waittime = 0;
-
-
- @Override
- public int getWaittime() {
- return waittime;
- }
-
- @Override
- public void setWaittime(int waittime) {
- this.waittime = waittime;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
deleted file mode 100644
index f6afae6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-public interface WaitTimeAccessor {
- int getWaittime();
- void setWaittime(int waittime);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
index 7e53d18..2809dfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -29,8 +29,6 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
@@ -484,8 +482,8 @@ public final class SliderUtils {
classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
} else {
log.info(
- "For faster submission of apps, upload dependencies using cmd {} {}",
- SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
+ "For faster submission of apps, upload dependencies using cmd " +
+ "enableFastLaunch");
}
classpath.addRemoteClasspathEnvVar();
classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
index 0498ebf..1f4581e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
@@ -19,10 +19,10 @@ package org.apache.hadoop.yarn.service.client;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.After;
@@ -36,7 +36,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
@@ -60,18 +59,10 @@ public class TestBuildExternalComponents {
// 2. check component names
private void buildAndCheckComponents(String appName, String appDef,
SliderFileSystem sfs, Set<String> names) throws Throwable {
- String[] args =
- { "build", ARG_FILE, ExampleAppJson.resourceName(appDef) };
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- ServiceCLI cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient();
- client.init(conf);
- client.start();
- }
- };
- cli.exec(clientArgs);
+ AppAdminClient client = AppAdminClient.createAppAdminClient(AppAdminClient
+ .DEFAULT_TYPE, conf);
+ client.actionSave(ExampleAppJson.resourceName(appDef), null, null,
+ null);
// verify generated conf
List<Component> components =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[58/58] [abbrv] hadoop git commit: YARN-7331. Change in few metrics
in new YARN UI related to native-services. Contributed by Sunil G
Posted by ji...@apache.org.
YARN-7331. Change in few metrics in new YARN UI related to native-services. Contributed by Sunil G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a288b0ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a288b0ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a288b0ac
Branch: refs/heads/yarn-native-services
Commit: a288b0accf8e0df0375cd5ca3aafdbfb142aa8c3
Parents: ba7ed7b
Author: Jian He <ji...@apache.org>
Authored: Mon Oct 30 13:16:19 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:20 2017 -0800
----------------------------------------------------------------------
.../main/webapp/app/adapters/yarn-servicedef.js | 2 +-
.../app/controllers/yarn-app/components.js | 8 ++---
.../yarn-component-instances/info.js | 2 +-
.../app/serializers/yarn-component-instance.js | 2 +-
.../app/templates/components/metrics-table.hbs | 37 +++-----------------
.../templates/yarn-component-instance/info.hbs | 6 +---
.../src/main/webapp/config/configs.env | 7 ----
.../src/main/webapp/config/default-config.js | 3 +-
8 files changed, 12 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
index dc5dbfd..3fb4a81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
@@ -20,7 +20,7 @@ import Ember from 'ember';
import RESTAbstractAdapter from './restabstract';
export default RESTAbstractAdapter.extend({
- address: "dashWebAddress",
+ address: "rmWebAddress",
restNameSpace: "dashService",
serverName: "DASH",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
index 947cb98..b4bac0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
@@ -30,7 +30,7 @@ export default Ember.Controller.extend({
cols.push({
id: 'name',
- headerTitle: 'Component Group',
+ headerTitle: 'Component',
contentPath: 'name',
cellComponentName: 'em-table-linked-cell',
getCellContent: function(row) {
@@ -49,13 +49,9 @@ export default Ember.Controller.extend({
contentPath: 'memory'
}, {
id: 'instances',
- headerTitle: '# Components',
+ headerTitle: 'Number Of Instances',
contentPath: 'instances',
observePath: true
- }, {
- id: 'createdDate',
- headerTitle: 'Created Time',
- contentPath: 'createdDate'
});
return ColumnDef.make(cols);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
index a676b34..0d1a4bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
@@ -31,7 +31,7 @@ export default Ember.Controller.extend({
cols.push({
id: 'instanceName',
- headerTitle: 'Component Name',
+ headerTitle: 'Component Instance',
contentPath: 'instanceName',
cellComponentName: 'em-table-linked-cell',
getCellContent: function(row) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
index 82eb273..1bd3b2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
@@ -28,7 +28,7 @@ export default DS.JSONAPISerializer.extend({
attributes: {
containerId: payload.id,
component: info.COMPONENT_NAME,
- instanceName: info.COMPONENT_NAME + '_' + payload.instanceId,
+ instanceName: info.COMPONENT_INSTANCE_NAME,
state: info.STATE,
createdTimestamp: payload.createdtime,
startedTimestamp: info.LAUNCH_TIME,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
index 6e4e990..17d38cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
@@ -20,7 +20,7 @@
<div class="row">
<div class="panel panel-default">
<div class="panel-heading">
- <div class="panel-title">{{type}} Metrics: Success Information</div>
+ <div class="panel-title">{{type}} Metrics: Detailed Information</div>
</div>
<div class="">
<table class="table table-hover table-custom-bordered table-custom-stripped table-radius-none table-border-none">
@@ -28,45 +28,18 @@
<tr>
<th>Desired Containers</th>
<th>Running Containers</th>
- <th>Completed Containers</th>
- <th>Pending Containers</th>
- <th>Surplus Containers</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td>{{metrics.ContainersDesired}}</td>
- <td>{{metrics.ContainersRunning}}</td>
- <td>{{metrics.ContainersCompleted}}</td>
- <td>{{metrics.ContainersPending}}</td>
- <td>{{metrics.SurplusContainers}}</td>
- </tr>
- </tbody>
- </table>
- </div>
- </div>
- </div>
- <div class="row">
- <div class="panel panel-default">
- <div class="panel-heading">
- <div class="panel-title">{{type}} Metrics: Failure Information</div>
- </div>
- <div class="">
- <table class="table table-hover table-custom-bordered table-custom-stripped table-radius-none table-border-none">
- <thead>
- <tr>
+ <th>Ready Containers</th>
<th>Failed Containers</th>
- <th>Containers Failed Since Last Threshold</th>
<th>Preempted Containers</th>
- <th>Pending Anti-Affinity Containers</th>
</tr>
</thead>
<tbody>
<tr>
+ <td>{{metrics.ContainersDesired}}</td>
+ <td>{{metrics.ContainersRunning}}</td>
+ <td>{{metrics.ContainersReady}}</td>
<td>{{metrics.ContainersFailed}}</td>
- <td>{{metrics.FailedSinceLastThreshold}}</td>
<td>{{metrics.ContainersPreempted}}</td>
- <td>{{metrics.PendingAAContainers}}</td>
</tr>
</tbody>
</table>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
index 3d5720e..5a94575 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
@@ -26,7 +26,7 @@
<table class="table table-striped table-bordered table-hover">
<tbody>
<tr>
- <td>Component Name</td>
+ <td>Component Instance</td>
<td>{{check-availability model.container.instanceName}}</td>
</tr>
<tr>
@@ -42,10 +42,6 @@
<td>{{check-availability model.container.state}}</td>
</tr>
<tr>
- <td>Created Time</td>
- <td>{{check-availability model.container.createdDate}}</td>
- </tr>
- <tr>
<td>Started Time</td>
<td>{{check-availability model.container.startedDate}}</td>
</tr>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env
index a795fc5..04577c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env
@@ -40,13 +40,6 @@ ENV = {
*/
//rmWebAddress: "localhost:8088",
- /*
- * Dash server web interface can be configured below.
- * By default dash web address is set as localhost:9191, uncomment and change
- * the following value for pointing to a different address.
- */
- //dashWebAddress: "localhost:9191",
-
/*
* Protocol scheme. It can be "http:" or "https:". By default, http is used.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
index d24e960..792f442 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
@@ -21,7 +21,6 @@ module.exports = { // YARN UI App configurations
localBaseAddress: "",
timelineWebAddress: "localhost:8188",
rmWebAddress: "localhost:8088",
- dashWebAddress: "localhost:9191",
protocolScheme: "http:"
},
namespaces: {
@@ -30,7 +29,7 @@ module.exports = { // YARN UI App configurations
cluster: 'ws/v1/cluster',
metrics: 'ws/v1/cluster/metrics',
timelineV2: 'ws/v2/timeline',
- dashService: 'services/v1/applications',
+ dashService: 'ws/v1/services',
node: '{nodeAddress}/ws/v1/node'
},
};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
deleted file mode 100644
index b3fdef9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Class to bond to a Jersey client, for UGI integration and SPNEGO.
- * <p>
- * Usage: create an instance, then when creating a Jersey <code>Client</code>
- * pass in to the constructor the handler provided by {@link #getHandler()}
- *
- * see <a href="https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html">Jersey docs</a>
- */
-public class UgiJerseyBinding implements
- HttpURLConnectionFactory {
- private static final Logger log =
- LoggerFactory.getLogger(UgiJerseyBinding.class);
-
- private final UrlConnectionOperations operations;
- private final URLConnectionClientHandler handler;
-
- /**
- * Construct an instance
- * @param operations operations instance
- */
- @SuppressWarnings("ThisEscapedInObjectConstruction")
- public UgiJerseyBinding(UrlConnectionOperations operations) {
- Preconditions.checkArgument(operations != null, "Null operations");
- this.operations = operations;
- handler = new URLConnectionClientHandler(this);
- }
-
- /**
- * Create an instance off the configuration. The SPNEGO policy
- * is derived from the current UGI settings.
- * @param conf config
- */
- public UgiJerseyBinding(Configuration conf) {
- this(new UrlConnectionOperations(conf));
- }
-
- /**
- * Get a URL connection.
- * @param url URL to connect to
- * @return the connection
- * @throws IOException any problem. {@link AuthenticationException}
- * errors are wrapped
- */
- @Override
- public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
- try {
- // open a connection handling status codes and so redirections
- // but as it opens a connection, it's less useful than you think.
-
- return operations.openConnection(url);
- } catch (AuthenticationException e) {
- throw new IOException(e);
- }
- }
-
- public UrlConnectionOperations getOperations() {
- return operations;
- }
-
- public URLConnectionClientHandler getHandler() {
- return handler;
- }
-
- /**
- * Get the SPNEGO flag (as found in the operations instance
- * @return the spnego policy
- */
- public boolean isUseSpnego() {
- return operations.isUseSpnego();
- }
-
-
- /**
- * Uprate error codes 400 and up into faults;
- * <p>
- * see {@link ExceptionConverter#convertJerseyException(String, String, UniformInterfaceException)}
- */
- public static IOException uprateFaults(HttpVerb verb, String url,
- UniformInterfaceException ex)
- throws IOException {
- return ExceptionConverter.convertJerseyException(verb.getVerb(),
- url, ex);
- }
-
- /**
- * Create the standard Jersey client Config
- * @return the recommended Jersey Client config
- */
- public ClientConfig createJerseyClientConfig() {
- ClientConfig clientConfig = new DefaultClientConfig();
- clientConfig.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, true);
- return clientConfig;
- }
-
- /**
- * Create a jersey client bonded to this handler, using the
- * supplied client config
- * @param clientConfig client configuratin
- * @return a new client instance to use
- */
- public Client createJerseyClient(ClientConfig clientConfig) {
- return new Client(getHandler(), clientConfig);
- }
-
- /**
- * Create a jersey client bonded to this handler, using the
- * client config created with {@link #createJerseyClientConfig()}
- * @return a new client instance to use
- */
- public Client createJerseyClient() {
- return createJerseyClient(createJerseyClientConfig());
- }
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java
deleted file mode 100644
index d7f768e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UrlConnectionOperations.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Operations on the JDK UrlConnection class.
- *
- */
-public class UrlConnectionOperations extends Configured {
- private static final Logger log =
- LoggerFactory.getLogger(UrlConnectionOperations.class);
-
- private SliderURLConnectionFactory connectionFactory;
-
- private boolean useSpnego = false;
-
- /**
- * Create an instance off the configuration. The SPNEGO policy
- * is derived from the current UGI settings.
- * @param conf config
- */
- public UrlConnectionOperations(Configuration conf) {
- super(conf);
- connectionFactory = SliderURLConnectionFactory.newInstance(conf);
- if (UserGroupInformation.isSecurityEnabled()) {
- log.debug("SPNEGO is enabled");
- setUseSpnego(true);
- }
- }
-
-
- public boolean isUseSpnego() {
- return useSpnego;
- }
-
- public void setUseSpnego(boolean useSpnego) {
- this.useSpnego = useSpnego;
- }
-
- /**
- * Opens a url with cache disabled, redirect handled in
- * (JDK) implementation.
- *
- * @param url to open
- * @return URLConnection
- * @throws IOException
- * @throws AuthenticationException authentication failure
- */
- public HttpURLConnection openConnection(URL url) throws
- IOException,
- AuthenticationException {
- Preconditions.checkArgument(url.getPort() != 0, "no port");
- return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java
deleted file mode 100644
index 98a76ea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/ServiceMonitor.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.component.ComponentState;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.RUNNING_BUT_UNREADY;
-import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_NOT_READY;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_READY;
-import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.READY;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_WINDOW;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEFAULT_READINESS_CHECK_INTERVAL;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.READINESS_CHECK_INTERVAL;
-
-public class ServiceMonitor extends AbstractService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceMonitor.class);
-
- public ScheduledExecutorService executorService;
- private Map<ContainerId, ComponentInstance> liveInstances = null;
- private ServiceContext context;
- private Configuration conf;
-
- public ServiceMonitor(String name, ServiceContext context) {
- super(name);
- liveInstances = context.scheduler.getLiveInstances();
- this.context = context;
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- executorService = Executors.newScheduledThreadPool(1);
- this.conf = conf;
- super.serviceInit(conf);
- }
-
- @Override
- public void serviceStart() throws Exception {
- long readinessCheckInterval = YarnServiceConf
- .getLong(READINESS_CHECK_INTERVAL, DEFAULT_READINESS_CHECK_INTERVAL,
- context.application.getConfiguration(), conf);
-
- executorService
- .scheduleAtFixedRate(new ReadinessChecker(), readinessCheckInterval,
- readinessCheckInterval, TimeUnit.SECONDS);
-
- // Default 6 hours.
- long failureResetInterval = YarnServiceConf
- .getLong(CONTAINER_FAILURE_WINDOW, 21600,
- context.application.getConfiguration(), conf);
-
- executorService
- .scheduleAtFixedRate(new ContainerFailureReset(), failureResetInterval,
- failureResetInterval, TimeUnit.SECONDS);
- }
-
- @Override
- public void serviceStop() throws Exception {
- if (executorService != null) {
- executorService.shutdownNow();
- }
- }
-
- private class ReadinessChecker implements Runnable {
-
- @Override
- public void run() {
-
- // check if the comp instance are ready
- for (Map.Entry<ContainerId, ComponentInstance> entry : liveInstances
- .entrySet()) {
- ComponentInstance instance = entry.getValue();
-
- ProbeStatus status = instance.ping();
- if (status.isSuccess()) {
- if (instance.getState() == RUNNING_BUT_UNREADY) {
- // synchronously update the state.
- instance.handle(
- new ComponentInstanceEvent(entry.getKey(), BECOME_READY));
- }
- } else {
- if (instance.getState() == READY) {
- instance.handle(
- new ComponentInstanceEvent(entry.getKey(), BECOME_NOT_READY));
- }
- }
- }
-
- for (Component component : context.scheduler.getAllComponents()
- .values()) {
- // If comp hasn't started yet and its dependencies are satisfied
- if (component.getState() == ComponentState.INIT && component
- .areDependenciesReady()) {
- LOG.info("[COMPONENT {}]: Dependencies satisfied, ramping up.",
- component.getName());
- ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
- .setDesired(component.getComponentSpec().getNumberOfContainers());
- component.handle(event);
- }
- }
- }
- }
-
- private class ContainerFailureReset implements Runnable {
- @Override
- public void run() {
- for (Component component : context.scheduler.getAllComponents().values()) {
- component.resetCompFailureCount();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java
deleted file mode 100644
index 10c1160..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/HttpProbe.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Map;
-
-public class HttpProbe extends Probe {
- protected static final Logger log = LoggerFactory.getLogger(HttpProbe.class);
-
- private static final String HOST_TOKEN = "${THIS_HOST}";
-
- private final String urlString;
- private final int timeout;
- private final int min, max;
-
-
- public HttpProbe(String url, int timeout, int min, int max, Configuration
- conf) {
- super("Http probe of " + url + " [" + min + "-" + max + "]", conf);
- this.urlString = url;
- this.timeout = timeout;
- this.min = min;
- this.max = max;
- }
-
- public static HttpProbe create(Map<String, String> props)
- throws IOException {
- String urlString = getProperty(props, WEB_PROBE_URL, null);
- new URL(urlString);
- int timeout = getPropertyInt(props, WEB_PROBE_CONNECT_TIMEOUT,
- WEB_PROBE_CONNECT_TIMEOUT_DEFAULT);
- int minSuccess = getPropertyInt(props, WEB_PROBE_MIN_SUCCESS,
- WEB_PROBE_MIN_SUCCESS_DEFAULT);
- int maxSuccess = getPropertyInt(props, WEB_PROBE_MAX_SUCCESS,
- WEB_PROBE_MAX_SUCCESS_DEFAULT);
- return new HttpProbe(urlString, timeout, minSuccess, maxSuccess, null);
- }
-
-
- private static HttpURLConnection getConnection(URL url, int timeout) throws
- IOException {
- HttpURLConnection connection = (HttpURLConnection) url.openConnection();
- connection.setInstanceFollowRedirects(true);
- connection.setConnectTimeout(timeout);
- return connection;
- }
-
- @Override
- public ProbeStatus ping(ComponentInstance instance) {
- ProbeStatus status = new ProbeStatus();
- ContainerStatus containerStatus = instance.getContainerStatus();
- if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
- || StringUtils.isEmpty(containerStatus.getHost())) {
- status.fail(this, new IOException("IP is not available yet"));
- return status;
- }
-
- String ip = containerStatus.getIPs().get(0);
- HttpURLConnection connection = null;
- try {
- URL url = new URL(urlString.replace(HOST_TOKEN, ip));
- connection = getConnection(url, this.timeout);
- int rc = connection.getResponseCode();
- if (rc < min || rc > max) {
- String error = "Probe " + url + " error code: " + rc;
- log.info(error);
- status.fail(this,
- new IOException(error));
- } else {
- status.succeed(this);
- }
- } catch (Throwable e) {
- String error = "Probe " + urlString + " failed for IP " + ip + ": " + e;
- log.info(error, e);
- status.fail(this,
- new IOException(error, e));
- } finally {
- if (connection != null) {
- connection.disconnect();
- }
- }
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java
deleted file mode 100644
index b575d69..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/LogEntryBuilder.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-/**
- * Build up log entries for ease of splunk
- */
-public class LogEntryBuilder {
-
- private final StringBuilder builder = new StringBuilder();
-
- public LogEntryBuilder() {
- }
-
- public LogEntryBuilder(String text) {
- elt(text);
- }
-
-
- public LogEntryBuilder(String name, Object value) {
- entry(name, value);
- }
-
- public LogEntryBuilder elt(String text) {
- addComma();
- builder.append(text);
- return this;
- }
-
- public LogEntryBuilder elt(String name, Object value) {
- addComma();
- entry(name, value);
- return this;
- }
-
- private void addComma() {
- if (!isEmpty()) {
- builder.append(", ");
- }
- }
-
- private void entry(String name, Object value) {
- builder.append(name).append('=');
- if (value != null) {
- builder.append('"').append(value.toString()).append('"');
- } else {
- builder.append("null");
- }
- }
-
- @Override
- public String toString() {
- return builder.toString();
- }
-
- private boolean isEmpty() {
- return builder.length() == 0;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java
deleted file mode 100644
index f5f3d99..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorKeys.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-/**
- * Config keys for monitoring
- */
-public interface MonitorKeys {
-
- /**
- * Port probing key : port to attempt to create a TCP connection to {@value}.
- */
- String PORT_PROBE_PORT = "port";
- /**
- * Port probing key : timeout for the the connection attempt {@value}.
- */
- String PORT_PROBE_CONNECT_TIMEOUT = "timeout";
- /**
- * Port probing default : timeout for the connection attempt {@value}.
- */
- int PORT_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
-
- /**
- * Web probing key : URL {@value}.
- */
- String WEB_PROBE_URL = "url";
- /**
- * Web probing key : min success code {@value}.
- */
- String WEB_PROBE_MIN_SUCCESS = "min.success";
- /**
- * Web probing key : max success code {@value}.
- */
- String WEB_PROBE_MAX_SUCCESS = "max.success";
- /**
- * Web probing default : min successful response code {@value}.
- */
- int WEB_PROBE_MIN_SUCCESS_DEFAULT = 200;
- /**
- * Web probing default : max successful response code {@value}.
- */
- int WEB_PROBE_MAX_SUCCESS_DEFAULT = 299;
- /**
- * Web probing key : timeout for the connection attempt {@value}
- */
- String WEB_PROBE_CONNECT_TIMEOUT = "timeout";
- /**
- * Port probing default : timeout for the connection attempt {@value}.
- */
- int WEB_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java
deleted file mode 100644
index 46d1fdb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/MonitorUtils.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Formatter;
-import java.util.Locale;
-
-/**
- * Various utils to work with the monitor
- */
-public final class MonitorUtils {
- protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
- .class);
-
- private MonitorUtils() {
- }
-
- public static String toPlural(int val) {
- return val != 1 ? "s" : "";
- }
-
- /**
- * Convert milliseconds to human time -the exact format is unspecified
- * @param milliseconds a time in milliseconds
- * @return a time that is converted to human intervals
- */
- public static String millisToHumanTime(long milliseconds) {
- StringBuilder sb = new StringBuilder();
- // Send all output to the Appendable object sb
- Formatter formatter = new Formatter(sb, Locale.US);
-
- long s = Math.abs(milliseconds / 1000);
- long m = Math.abs(milliseconds % 1000);
- if (milliseconds > 0) {
- formatter.format("%d.%03ds", s, m);
- } else if (milliseconds == 0) {
- formatter.format("0");
- } else {
- formatter.format("-%d.%03ds", s, m);
- }
- return sb.toString();
- }
-
- public static Probe getProbe(ReadinessCheck readinessCheck) {
- if (readinessCheck == null) {
- return null;
- }
- if (readinessCheck.getType() == null) {
- return null;
- }
- try {
- switch (readinessCheck.getType()) {
- case HTTP:
- return HttpProbe.create(readinessCheck.getProps());
- case PORT:
- return PortProbe.create(readinessCheck.getProps());
- default:
- return null;
- }
- } catch (Throwable t) {
- throw new IllegalArgumentException("Error creating readiness check " +
- t);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java
deleted file mode 100644
index f6cf3ae..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/PortProbe.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.util.Map;
-
-/**
- * Probe for a port being open.
- */
-public class PortProbe extends Probe {
- protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
- private final int port;
- private final int timeout;
-
- public PortProbe(int port, int timeout) {
- super("Port probe of " + port + " for " + timeout + "ms", null);
- this.port = port;
- this.timeout = timeout;
- }
-
- public static PortProbe create(Map<String, String> props)
- throws IOException {
- int port = getPropertyInt(props, PORT_PROBE_PORT, null);
-
- if (port >= 65536) {
- throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
- "range");
- }
-
- int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
- PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
-
- return new PortProbe(port, timeout);
- }
-
- /**
- * Try to connect to the (host,port); a failure to connect within
- * the specified timeout is a failure.
- * @param instance role instance
- * @return the outcome
- */
- @Override
- public ProbeStatus ping(ComponentInstance instance) {
- ProbeStatus status = new ProbeStatus();
-
- if (instance.getContainerStatus() == null || SliderUtils
- .isEmpty(instance.getContainerStatus().getIPs())) {
- status.fail(this, new IOException(
- instance.getCompInstanceName() + ": IP is not available yet"));
- return status;
- }
-
- String ip = instance.getContainerStatus().getIPs().get(0);
- InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
- Socket socket = new Socket();
- try {
- if (log.isDebugEnabled()) {
- log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
- .toString() + ", timeout=" + MonitorUtils
- .millisToHumanTime(timeout));
- }
- socket.connect(sockAddr, timeout);
- status.succeed(this);
- } catch (Throwable e) {
- String error =
- instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
- log.debug(error, e);
- status.fail(this, new IOException(error, e));
- } finally {
- IOUtils.closeSocket(socket);
- }
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
deleted file mode 100644
index b851fb7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Base class of all probes.
- */
-public abstract class Probe implements MonitorKeys {
-
- protected final Configuration conf;
- private String name;
-
- /**
- * Create a probe of a specific name
- *
- * @param name probe name
- * @param conf configuration being stored.
- */
- public Probe(String name, Configuration conf) {
- this.name = name;
- this.conf = conf;
- }
-
-
- protected void setName(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
-
-
- @Override
- public String toString() {
- return getName();
- }
-
- public static String getProperty(Map<String, String> props, String name,
- String defaultValue) throws IOException {
- String value = props.get(name);
- if (StringUtils.isEmpty(value)) {
- if (defaultValue == null) {
- throw new IOException(name + " not specified");
- }
- return defaultValue;
- }
- return value;
- }
-
- public static int getPropertyInt(Map<String, String> props, String name,
- Integer defaultValue) throws IOException {
- String value = props.get(name);
- if (StringUtils.isEmpty(value)) {
- if (defaultValue == null) {
- throw new IOException(name + " not specified");
- }
- return defaultValue;
- }
- return Integer.parseInt(value);
- }
-
- /**
- * perform any prelaunch initialization
- */
- public void init() throws IOException {
-
- }
-
- /**
- * Ping the endpoint. All exceptions must be caught and included in the
- * (failure) status.
- *
- * @param instance instance to ping
- * @return the status
- */
- public abstract ProbeStatus ping(ComponentInstance instance);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
deleted file mode 100644
index 7cd761c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.servicemonitor.probe;
-
-import java.io.Serializable;
-import java.util.Date;
-
-/**
- * Status message of a probe. This is designed to be sent over the wire, though the exception
- * Had better be unserializable at the far end if that is to work.
- */
-public final class ProbeStatus implements Serializable {
- private static final long serialVersionUID = 165468L;
-
- private long timestamp;
- private String timestampText;
- private boolean success;
- private boolean realOutcome;
- private String message;
- private Throwable thrown;
- private transient Probe originator;
-
- public ProbeStatus() {
- }
-
- public ProbeStatus(long timestamp, String message, Throwable thrown) {
- this.success = false;
- this.message = message;
- this.thrown = thrown;
- setTimestamp(timestamp);
- }
-
- public ProbeStatus(long timestamp, String message) {
- this.success = true;
- setTimestamp(timestamp);
- this.message = message;
- this.thrown = null;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- timestampText = new Date(timestamp).toString();
- }
-
- public boolean isSuccess() {
- return success;
- }
-
- /**
- * Set both the success and the real outcome bits to the same value
- * @param success the new value
- */
- public void setSuccess(boolean success) {
- this.success = success;
- realOutcome = success;
- }
-
- public String getTimestampText() {
- return timestampText;
- }
-
- public boolean getRealOutcome() {
- return realOutcome;
- }
-
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- }
-
- public Throwable getThrown() {
- return thrown;
- }
-
- public void setThrown(Throwable thrown) {
- this.thrown = thrown;
- }
-
- /**
- * Get the probe that generated this result. May be null
- * @return a possibly null reference to a probe
- */
- public Probe getOriginator() {
- return originator;
- }
-
- /**
- * The probe has succeeded -capture the current timestamp, set
- * success to true, and record any other data needed.
- * @param probe probe
- */
- public void succeed(Probe probe) {
- finish(probe, true, probe.getName(), null);
- }
-
- /**
- * A probe has failed either because the test returned false, or an exception
- * was thrown. The {@link #success} field is set to false, any exception
- * thrown is recorded.
- * @param probe probe that failed
- * @param thrown an exception that was thrown.
- */
- public void fail(Probe probe, Throwable thrown) {
- finish(probe, false, "Failure in " + probe, thrown);
- }
-
- public void finish(Probe probe, boolean succeeded, String text, Throwable thrown) {
- setTimestamp(System.currentTimeMillis());
- setSuccess(succeeded);
- originator = probe;
- message = text;
- this.thrown = thrown;
- }
-
- @Override
- public String toString() {
- LogEntryBuilder builder = new LogEntryBuilder("Probe Status");
- builder.elt("time", timestampText)
- .elt("outcome", (success ? "success" : "failure"));
-
- if (success != realOutcome) {
- builder.elt("originaloutcome", (realOutcome ? "success" : "failure"));
- }
- builder.elt("message", message);
- if (thrown != null) {
- builder.elt("exception", thrown);
- }
-
- return builder.toString();
- }
-
- /**
- * Flip the success bit on while the real outcome bit is kept false
- */
- public void markAsSuccessful() {
- success = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
deleted file mode 100644
index cf4e836..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceMetricsSink.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-import org.apache.commons.configuration2.SubsetConfiguration;
-import org.apache.hadoop.metrics2.MetricsRecord;
-import org.apache.hadoop.metrics2.MetricsSink;
-import org.apache.hadoop.metrics2.MetricsTag;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Write the metrics to a ATSv2. Generally, this class is instantiated via
- * hadoop-metrics2 property files. Specifically, you would create this class by
- * adding the following to by This would actually be set as: <code>
- * [prefix].sink.[some instance name].class
- * =org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink
- * </code>, where <tt>prefix</tt> is "atsv2": and <tt>some instance name</tt> is
- * just any unique name, so properties can be differentiated if there are
- * multiple sinks of the same type created
- */
-public class ServiceMetricsSink implements MetricsSink {
-
- private static final Logger log =
- LoggerFactory.getLogger(ServiceMetricsSink.class);
-
- private ServiceTimelinePublisher serviceTimelinePublisher;
-
- public ServiceMetricsSink() {
-
- }
-
- public ServiceMetricsSink(ServiceTimelinePublisher publisher) {
- serviceTimelinePublisher = publisher;
- }
-
- /**
- * Publishes service and component metrics to ATS.
- */
- @Override
- public void putMetrics(MetricsRecord record) {
- if (serviceTimelinePublisher.isStopped()) {
- log.warn("ServiceTimelinePublisher has stopped. "
- + "Not publishing any more metrics to ATS.");
- return;
- }
-
- boolean isServiceMetrics = false;
- boolean isComponentMetrics = false;
- String appId = null;
- for (MetricsTag tag : record.tags()) {
- if (tag.name().equals("type") && tag.value().equals("service")) {
- isServiceMetrics = true;
- } else if (tag.name().equals("type") && tag.value().equals("component")) {
- isComponentMetrics = true;
- break; // if component metrics, no more information required from tag so
- // break the loop
- } else if (tag.name().equals("appId")) {
- appId = tag.value();
- }
- }
-
- if (isServiceMetrics && appId != null) {
- if (log.isDebugEnabled()) {
- log.debug("Publishing service metrics. " + record);
- }
- serviceTimelinePublisher.publishMetrics(record.metrics(), appId,
- ServiceTimelineEntityType.SERVICE_ATTEMPT.toString(),
- record.timestamp());
- } else if (isComponentMetrics) {
- if (log.isDebugEnabled()) {
- log.debug("Publishing Component metrics. " + record);
- }
- serviceTimelinePublisher.publishMetrics(record.metrics(), record.name(),
- ServiceTimelineEntityType.COMPONENT.toString(), record.timestamp());
- }
- }
-
- @Override
- public void init(SubsetConfiguration conf) {
- }
-
- @Override
- public void flush() {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
deleted file mode 100644
index d5c9539..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEntityType.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-/**
- * Slider entities that are published to ATS.
- */
-public enum ServiceTimelineEntityType {
- /**
- * Used for publishing service entity information.
- */
- SERVICE_ATTEMPT,
-
- /**
- * Used for publishing component entity information.
- */
- COMPONENT,
-
- /**
- * Used for publishing component instance entity information.
- */
- COMPONENT_INSTANCE
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
deleted file mode 100644
index 7f7f9a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-/**
- * Events that are used to store in ATS.
- */
-public enum ServiceTimelineEvent {
- SERVICE_ATTEMPT_REGISTERED,
-
- SERVICE_ATTEMPT_UNREGISTERED,
-
- COMPONENT_INSTANCE_REGISTERED,
-
- COMPONENT_INSTANCE_UNREGISTERED,
-
- COMPONENT_INSTANCE_UPDATED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
deleted file mode 100644
index 78a7171..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineMetricsConstants.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-/**
- * Constants which are stored as key in ATS
- */
-public final class ServiceTimelineMetricsConstants {
-
- public static final String URI = "URI";
-
- public static final String NAME = "NAME";
-
- public static final String STATE = "STATE";
-
- public static final String EXIT_STATUS_CODE = "EXIT_STATUS_CODE";
-
- public static final String EXIT_REASON = "EXIT_REASON";
-
- public static final String DIAGNOSTICS_INFO = "DIAGNOSTICS_INFO";
-
- public static final String LAUNCH_TIME = "LAUNCH_TIME";
-
- public static final String QUICK_LINKS = "QUICK_LINKS";
-
- public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND";
-
- public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS";
-
- public static final String RUNNING_CONTAINERS =
- "NUMBER_OF_RUNNING_CONTAINERS";
-
- /**
- * Artifacts constants.
- */
- public static final String ARTIFACT_ID = "ARTIFACT_ID";
-
- public static final String ARTIFACT_TYPE = "ARTIFACT_TYPE";
-
- public static final String ARTIFACT_URI = "ARTIFACT_URI";
-
- /**
- * Resource constants.
- */
- public static final String RESOURCE_CPU = "RESOURCE_CPU";
-
- public static final String RESOURCE_MEMORY = "RESOURCE_MEMORY";
-
- public static final String RESOURCE_PROFILE = "RESOURCE_PROFILE";
-
- /**
- * component instance constants.
- */
- public static final String IP = "IP";
-
- public static final String HOSTNAME = "HOSTNAME";
-
- public static final String BARE_HOST = "BARE_HOST";
-
- public static final String COMPONENT_NAME = "COMPONENT_NAME";
-
- public static final String COMPONENT_INSTANCE_NAME = "COMPONENT_INSTANCE_NAME";
-
- /**
- * component constants.
- */
- public static final String DEPENDENCIES = "DEPENDENCIES";
-
- public static final String DESCRIPTION = "DESCRIPTION";
-
- public static final String RUN_PRIVILEGED_CONTAINER =
- "RUN_PRIVILEGED_CONTAINER";
-
- public static final String PLACEMENT_POLICY = "PLACEMENT_POLICY";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
deleted file mode 100644
index 243baea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.timelineservice;
-
-import org.apache.hadoop.metrics2.AbstractMetric;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.timelineservice.ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO;
-
-/**
- * A single service that publishes all the Timeline Entities.
- */
-public class ServiceTimelinePublisher extends CompositeService {
-
- // Number of bytes of config which can be published in one shot to ATSv2.
- public static final int ATS_CONFIG_PUBLISH_SIZE_BYTES = 10 * 1024;
-
- private TimelineV2Client timelineClient;
-
- private volatile boolean stopped = false;
-
- private static final Logger log =
- LoggerFactory.getLogger(ServiceTimelinePublisher.class);
-
- @Override
- protected void serviceInit(org.apache.hadoop.conf.Configuration configuration)
- throws Exception {
- addService(timelineClient);
- }
-
-
- @Override
- protected void serviceStop() throws Exception {
- stopped = true;
- super.serviceStop();
- }
-
- public boolean isStopped() {
- return stopped;
- }
-
- public ServiceTimelinePublisher(TimelineV2Client client) {
- super(ServiceTimelinePublisher.class.getName());
- timelineClient = client;
- }
-
- public void serviceAttemptRegistered(Application application,
- org.apache.hadoop.conf.Configuration systemConf) {
- long currentTimeMillis = application.getLaunchTime() == null
- ? System.currentTimeMillis() : application.getLaunchTime().getTime();
-
- TimelineEntity entity = createServiceAttemptEntity(application.getId());
- entity.setCreatedTime(currentTimeMillis);
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.NAME, application.getName());
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- application.getState().toString());
- entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
- currentTimeMillis);
- entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
- application.getQuicklinks());
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent.setId(ServiceTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString());
- startEvent.setTimestamp(currentTimeMillis);
- entity.addEvent(startEvent);
-
- // publish before configurations published
- putEntity(entity);
-
- // publish system config - YarnConfiguration
- populateTimelineEntity(systemConf.iterator(), application.getId(),
- ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
- // publish user conf
- publishUserConf(application.getConfiguration(), application.getId(),
- ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
-
- // publish component as separate entity.
- publishComponents(application.getComponents());
- }
-
- public void serviceAttemptUpdated(Application application) {
- TimelineEntity entity = createServiceAttemptEntity(application.getId());
- entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
- application.getQuicklinks());
- putEntity(entity);
- }
-
- public void serviceAttemptUnregistered(ServiceContext context,
- String diagnostics) {
- TimelineEntity entity = createServiceAttemptEntity(
- context.attemptId.getApplicationId().toString());
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- FinalApplicationStatus.ENDED);
- entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent finishEvent = new TimelineEvent();
- finishEvent
- .setId(ServiceTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString());
- finishEvent.setTimestamp(System.currentTimeMillis());
- entity.addEvent(finishEvent);
-
- putEntity(entity);
- }
-
- public void componentInstanceStarted(Container container,
- ComponentInstance instance) {
-
- TimelineEntity entity = createComponentInstanceEntity(container.getId());
- entity.setCreatedTime(container.getLaunchTime().getTime());
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.BARE_HOST,
- container.getBareHost());
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- container.getState().toString());
- entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
- container.getLaunchTime().getTime());
- entityInfos.put(ServiceTimelineMetricsConstants.COMPONENT_NAME,
- instance.getCompName());
- entityInfos.put(ServiceTimelineMetricsConstants.COMPONENT_INSTANCE_NAME,
- instance.getCompInstanceName());
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent
- .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_REGISTERED.toString());
- startEvent.setTimestamp(container.getLaunchTime().getTime());
- entity.addEvent(startEvent);
-
- putEntity(entity);
- }
-
- public void componentInstanceFinished(ComponentInstance instance,
- int exitCode, ContainerState state, String diagnostics) {
- TimelineEntity entity = createComponentInstanceEntity(
- instance.getContainer().getId().toString());
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE,
- exitCode);
- entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
- entityInfos.put(ServiceTimelineMetricsConstants.STATE, state);
- entity.addInfo(entityInfos);
-
- // add an event
- TimelineEvent startEvent = new TimelineEvent();
- startEvent
- .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UNREGISTERED.toString());
- startEvent.setTimestamp(System.currentTimeMillis());
- entity.addEvent(startEvent);
-
- putEntity(entity);
- }
-
- public void componentInstanceUpdated(Container container) {
- TimelineEntity entity = createComponentInstanceEntity(container.getId());
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.IP, container.getIp());
- entityInfos.put(ServiceTimelineMetricsConstants.HOSTNAME,
- container.getHostname());
- entityInfos.put(ServiceTimelineMetricsConstants.STATE,
- container.getState().toString());
- entity.addInfo(entityInfos);
-
- TimelineEvent updateEvent = new TimelineEvent();
- updateEvent
- .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UPDATED.toString());
- updateEvent.setTimestamp(System.currentTimeMillis());
- entity.addEvent(updateEvent);
-
- putEntity(entity);
- }
-
- private void publishComponents(List<Component> components) {
- long currentTimeMillis = System.currentTimeMillis();
- for (Component component : components) {
- TimelineEntity entity = createComponentEntity(component.getName());
- entity.setCreatedTime(currentTimeMillis);
-
- // create info keys
- Map<String, Object> entityInfos = new HashMap<String, Object>();
- entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_ID,
- component.getArtifact().getId());
- entityInfos.put(ServiceTimelineMetricsConstants.ARTIFACT_TYPE,
- component.getArtifact().getType().toString());
- if (component.getResource().getProfile() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_PROFILE,
- component.getResource().getProfile());
- }
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_CPU,
- component.getResource().getCpus());
- entityInfos.put(ServiceTimelineMetricsConstants.RESOURCE_MEMORY,
- component.getResource().getMemory());
-
- if (component.getLaunchCommand() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_COMMAND,
- component.getLaunchCommand());
- }
- entityInfos.put(ServiceTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER,
- component.getRunPrivilegedContainer().toString());
- if (component.getPlacementPolicy() != null) {
- entityInfos.put(ServiceTimelineMetricsConstants.PLACEMENT_POLICY,
- component.getPlacementPolicy().getLabel());
- }
- entity.addInfo(entityInfos);
-
- putEntity(entity);
-
- // publish component specific configurations
- publishUserConf(component.getConfiguration(), component.getName(),
- ServiceTimelineEntityType.COMPONENT.toString());
- }
- }
-
- private void publishUserConf(Configuration configuration,
- String entityId, String entityType) {
- populateTimelineEntity(configuration.getProperties().entrySet().iterator(),
- entityId, entityType);
-
- populateTimelineEntity(configuration.getEnv().entrySet().iterator(),
- entityId, entityType);
-
- for (ConfigFile configFile : configuration.getFiles()) {
- populateTimelineEntity(configFile.getProps().entrySet().iterator(),
- entityId, entityType);
- }
- }
-
- private void populateTimelineEntity(Iterator<Entry<String, String>> iterator,
- String entityId, String entityType) {
- int configSize = 0;
- TimelineEntity entity = createTimelineEntity(entityId, entityType);
- while (iterator.hasNext()) {
- Entry<String, String> entry = iterator.next();
- int size = entry.getKey().length() + entry.getValue().length();
- configSize += size;
- // Configs are split into multiple entities if they exceed 100kb in size.
- if (configSize > ATS_CONFIG_PUBLISH_SIZE_BYTES) {
- if (entity.getConfigs().size() > 0) {
- putEntity(entity);
- entity = createTimelineEntity(entityId, entityType);
- }
- configSize = size;
- }
- entity.addConfig(entry.getKey(), entry.getValue());
- }
- if (configSize > 0) {
- putEntity(entity);
- }
- }
-
- /**
- * Called from ServiceMetricsSink at regular interval of time.
- * @param metrics of service or components
- * @param entityId Id of entity
- * @param entityType Type of entity
- * @param timestamp
- */
- public void publishMetrics(Iterable<AbstractMetric> metrics, String entityId,
- String entityType, long timestamp) {
- TimelineEntity entity = createTimelineEntity(entityId, entityType);
- Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
- for (AbstractMetric metric : metrics) {
- TimelineMetric timelineMetric = new TimelineMetric();
- timelineMetric.setId(metric.name());
- timelineMetric.addValue(timestamp, metric.value());
- entityMetrics.add(timelineMetric);
- }
- entity.setMetrics(entityMetrics);
- putEntity(entity);
- }
-
- private TimelineEntity createServiceAttemptEntity(String serviceId) {
- TimelineEntity entity = createTimelineEntity(serviceId,
- ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
- return entity;
- }
-
- private TimelineEntity createComponentInstanceEntity(String instanceId) {
- TimelineEntity entity = createTimelineEntity(instanceId,
- ServiceTimelineEntityType.COMPONENT_INSTANCE.toString());
- return entity;
- }
-
- private TimelineEntity createComponentEntity(String componentId) {
- TimelineEntity entity = createTimelineEntity(componentId,
- ServiceTimelineEntityType.COMPONENT.toString());
- return entity;
- }
-
- private TimelineEntity createTimelineEntity(String entityId,
- String entityType) {
- TimelineEntity entity = new TimelineEntity();
- entity.setId(entityId);
- entity.setType(entityType);
- return entity;
- }
-
- private void putEntity(TimelineEntity entity) {
- try {
- if (log.isDebugEnabled()) {
- log.debug("Publishing the entity " + entity + ", JSON-style content: "
- + TimelineUtils.dumpTimelineRecordtoJSON(entity));
- }
- if (timelineClient != null) {
- timelineClient.putEntitiesAsync(entity);
- } else {
- log.error("Seems like client has been removed before the entity "
- + "could be published for " + entity);
- }
- } catch (Exception e) {
- log.error("Error when publishing entity " + entity, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
deleted file mode 100644
index 72f7842..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * ATS implementation
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.service.timelineservice;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
deleted file mode 100644
index 2607c08..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ApplicationReportSerDeser.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.map.JsonMappingException;
-
-import java.io.IOException;
-
-/**
- * Persistence of {@link SerializedApplicationReport}
- *
- */
-public class ApplicationReportSerDeser
- extends JsonSerDeser<SerializedApplicationReport> {
- public ApplicationReportSerDeser() {
- super(SerializedApplicationReport.class);
- }
-
-
- private static final ApplicationReportSerDeser
- staticinstance = new ApplicationReportSerDeser();
-
- /**
- * Convert an instance to a JSON string -sync access to a shared ser/deser
- * object instance
- * @param instance object to convert
- * @return a JSON string description
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public static String toString(SerializedApplicationReport instance)
- throws IOException, JsonGenerationException, JsonMappingException {
- synchronized (staticinstance) {
- return staticinstance.toJson(instance);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
deleted file mode 100644
index 86896b2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.encodeForRegistry;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.convertUsername;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.getCurrentUsernameUnencoded;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
-
-/**
- * Generic code to get the URLs for clients via the registry
- */
-public class ClientRegistryBinder {
- private static final Logger log =
- LoggerFactory.getLogger(ClientRegistryBinder.class);
-
- private final RegistryOperations operations;
-
- public ClientRegistryBinder(RegistryOperations operations) {
- this.operations = operations;
- }
-
- /**
- * Buld the user path -switches to the system path if the user is "".
- * It also cross-converts the username to ascii via punycode
- * @param username username or ""
- * @return the path to the user
- */
- public static String homePathForUser(String username) {
- Preconditions.checkArgument(username != null, "null user");
-
- // catch recursion
- if (username.startsWith(RegistryConstants.PATH_USERS)) {
- return username;
- }
-
- if (username.isEmpty()) {
- return RegistryConstants.PATH_SYSTEM_SERVICES;
- }
-
- // convert username to registry name
- String convertedName = convertUsername(username);
-
- return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
- encodeForRegistry(convertedName));
- }
-
- /**
- * Get the current username, before any encoding has been applied.
- * @return the current user from the kerberos identity, falling back
- * to the user and/or env variables.
- */
- public static String currentUsernameUnencoded() {
- String env_hadoop_username = System.getenv(
- RegistryInternalConstants.HADOOP_USER_NAME);
- return getCurrentUsernameUnencoded(env_hadoop_username);
- }
-
- /**
- * Qualify a user.
- * <ol>
- * <li> <code>"~"</code> maps to user home path home</li>
- * <li> <code>"~user"</code> maps to <code>/users/$user</code></li>
- * <li> <code>"/"</code> maps to <code>/services/</code></li>
- * </ol>
- * @param user the username
- * @return the base path
- */
- public static String qualifyUser(String user) {
- // qualify the user
- String t = user.trim();
- if (t.startsWith("/")) {
- // already resolved
- return t;
- } else if (t.equals("~")) {
- // self
- return currentUsernameUnencoded();
- } else if (t.startsWith("~")) {
- // another user
- // convert username to registry name
- String convertedName = convertUsername(t.substring(1));
-
- return RegistryPathUtils.join(RegistryConstants.PATH_USERS,
- encodeForRegistry(convertedName));
- } else {
- return "/" + t;
- }
- }
-
- /**
- * Look up an external REST API
- * @param user user which will be qualified as per {@link #qualifyUser(String)}
- * @param serviceClass service class
- * @param instance instance name
- * @param api API
- * @return the API, or an exception is raised.
- * @throws IOException
- */
- public String lookupExternalRestAPI(String user,
- String serviceClass,
- String instance,
- String api)
- throws IOException {
- String qualified = qualifyUser(user);
- String path = servicePath(qualified, serviceClass, instance);
- String restAPI = resolveExternalRestAPI(api, path);
- if (restAPI == null) {
- throw new PathNotFoundException(path + " API " + api);
- }
- return restAPI;
- }
-
- /**
- * Resolve a service record then return an external REST API exported it.
- *
- * @param api API to resolve
- * @param path path of the service record
- * @return null if the record exists but the API is absent or it has no
- * REST endpoints.
- * @throws IOException resolution problems, as covered in
- * {@link RegistryOperations#resolve(String)}
- */
- protected String resolveExternalRestAPI(String api, String path) throws
- IOException {
- ServiceRecord record = operations.resolve(path);
- return lookupRestAPI(record, api, true);
- }
-
- /**
- * Look up an external REST API endpoint
- * @param record service record
- * @param api URI of api
- * @param external flag to indicate this is an external record
- * @return the first endpoint of the implementation, or null if there
- * is no entry for the API, implementation or it's the wrong type.
- */
- public static String lookupRestAPI(ServiceRecord record,
- String api, boolean external) throws InvalidRecordException {
- try {
- String url = null;
- Endpoint endpoint = getEndpoint(record, api, external);
- List<String> addresses =
- RegistryTypeUtils.retrieveAddressesUriType(endpoint);
- if (addresses != null && !addresses.isEmpty()) {
- url = addresses.get(0);
- }
- return url;
- } catch (InvalidRecordException e) {
- log.debug("looking for API {}", api, e);
- return null;
- }
- }
-
- /**
- * Get an endpont by API
- * @param record service record
- * @param api API
- * @param external flag to indicate this is an external record
- * @return the endpoint or null
- */
- public static Endpoint getEndpoint(ServiceRecord record,
- String api,
- boolean external) {
- return external ? record.getExternalEndpoint(api)
- : record.getInternalEndpoint(api);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
deleted file mode 100644
index 9f0e5d4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/Comparators.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import java.io.Serializable;
-import java.util.Comparator;
-
-/**
- * Some general comparators
- */
-public class Comparators {
-
- public static class LongComparator implements Comparator<Long>, Serializable {
- @Override
- public int compare(Long o1, Long o2) {
- return o1.compareTo(o2);
- }
- }
-
- public static class InvertedLongComparator
- implements Comparator<Long>, Serializable {
- @Override
- public int compare(Long o1, Long o2) {
- return o2.compareTo(o1);
- }
- }
-
- /**
- * Little template class to reverse any comparitor
- * @param <CompareType> the type that is being compared
- */
- public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
- Serializable {
-
- final Comparator<CompareType> instance;
-
- public ComparatorReverser(Comparator<CompareType> instance) {
- this.instance = instance;
- }
-
- @Override
- public int compare(CompareType first, CompareType second) {
- return instance.compare(second, first);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/58] [abbrv] hadoop git commit: YARN-7073. Yarn native services
rest API documentation. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-7073. Yarn native services rest API documentation. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dd4490a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dd4490a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dd4490a
Branch: refs/heads/yarn-native-services
Commit: 7dd4490a44442bd2b682a5f1a8605e6f6c053037
Parents: bd96c4c
Author: Jian He <ji...@apache.org>
Authored: Thu Aug 31 17:17:22 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:14 2017 -0800
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 1 +
.../hadoop/yarn/service/webapp/ApiServer.java | 47 +-
.../definition/YARN-Services-Examples.md | 245 ++++++++
...RN-Simplified-V1-API-Layer-For-Services.yaml | 128 ++--
.../native-services/NativeServicesAPI.md | 606 +++++++++++++++++++
5 files changed, 973 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd4490a/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 8cb57c8..ad893a2 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -157,6 +157,7 @@
<menu name="YARN Native Services" inherit="top">
<item name="Introduction" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesIntro.html"/>
+ <item name="Native Services API" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesAPI.html"/>
<item name="Native Services Discovery" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesDiscovery.html"/>
</menu>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd4490a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index f55e3f1..e8286ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -24,12 +24,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,6 +48,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.Map;
+import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
/**
@@ -76,11 +76,11 @@ public class ApiServer {
@GET
@Path(VERSION)
@Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+ @Produces({ MediaType.APPLICATION_JSON })
public Response getVersion() {
String version = VersionInfo.getBuildVersion();
LOG.info(version);
- return Response.ok(version).build();
+ return Response.ok("{ \"hadoop_version\": \"" + version + "\"}").build();
}
@POST
@@ -94,11 +94,11 @@ public class ApiServer {
ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service);
LOG.info("Successfully created service " + service.getName()
+ " applicationId = " + applicationId);
- serviceStatus.setState(ServiceState.ACCEPTED);
+ serviceStatus.setState(ACCEPTED);
serviceStatus.setUri(
CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
.getName());
- return Response.status(Status.CREATED).entity(serviceStatus).build();
+ return Response.status(Status.ACCEPTED).entity(serviceStatus).build();
} catch (IllegalArgumentException e) {
serviceStatus.setDiagnostics(e.getMessage());
return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
@@ -182,16 +182,16 @@ public class ApiServer {
+ ": Invalid number of containers specified " + component
.getNumberOfContainers()).build();
}
+ ServiceStatus status = new ServiceStatus();
try {
Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
Collections.singletonMap(component.getName(),
component.getNumberOfContainers()));
- return Response.ok().entity(
- "Updating component " + componentName + " size from " + original
- .get(componentName) + " to " + component.getNumberOfContainers())
- .build();
+ status.setDiagnostics(
+ "Updating component (" + componentName + ") size from " + original
+ .get(componentName) + " to " + component.getNumberOfContainers());
+ return Response.ok().entity(status).build();
} catch (YarnException | IOException e) {
- ServiceStatus status = new ServiceStatus();
status.setDiagnostics(e.getMessage());
return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
.build();
@@ -244,31 +244,40 @@ public class ApiServer {
}
private Response updateLifetime(String appName, Service updateAppData) {
+ ServiceStatus status = new ServiceStatus();
try {
String newLifeTime =
SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
- return Response.ok("Service " + appName + " lifeTime is successfully updated to "
- + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
+ status.setDiagnostics(
+ "Service (" + appName + ")'s lifeTime is updated to " + newLifeTime
+ + ", " + updateAppData.getLifetime()
+ + " seconds remaining");
+ return Response.ok(status).build();
} catch (Exception e) {
String message =
- "Failed to update service (" + appName + ") lifetime ("
- + updateAppData.getLifetime() + ")";
+ "Failed to update service (" + appName + ")'s lifetime to "
+ + updateAppData.getLifetime();
LOG.error(message, e);
- return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + " : " + e.getMessage()).build();
+ status.setDiagnostics(message + ": " + e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
+ .build();
}
}
private Response startService(String appName) {
+ ServiceStatus status = new ServiceStatus();
try {
SERVICE_CLIENT.actionStart(appName);
LOG.info("Successfully started service " + appName);
- return Response.ok("Service " + appName + " is successfully started").build();
+ status.setDiagnostics("Service " + appName + " is successfully started.");
+ status.setState(ServiceState.ACCEPTED);
+ return Response.ok(status).build();
} catch (Exception e) {
String message = "Failed to start service " + appName;
+ status.setDiagnostics(message + ": " + e.getMessage());
LOG.info(message, e);
return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(message + ": " + e.getMessage()).build();
+ .entity(status).build();
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd4490a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
new file mode 100644
index 0000000..3cd3d48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
@@ -0,0 +1,245 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+## Examples
+
+### Create a simple single-component service with most attribute values as defaults
+POST URL - http://localhost:9191/ws/v1/services
+
+##### POST Request JSON
+```json
+{
+ "name": "hello-world",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "nginx:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "./start_nginx.sh",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+
+##### GET Response JSON
+GET URL - http://localhost:9191/ws/v1/services/hello-world
+
+Note, lifetime value of -1 means unlimited lifetime.
+
+```json
+{
+ "name": "hello-world",
+ "id": "application_1503963985568_0002",
+ "lifetime": -1,
+ "components": [
+ {
+ "name": "hello",
+ "dependencies": [],
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": [],
+ "containers": [
+ {
+ "id": "container_e03_1503963985568_0002_01_000001",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051512412,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-0"
+ },
+ {
+ "id": "container_e03_1503963985568_0002_01_000002",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051536450,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-1"
+ }
+ ],
+ "launch_command": "./start_nginx.sh",
+ "number_of_containers": 1,
+ "run_privileged_container": false
+ }
+ ],
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": {}
+}
+
+```
+### Update to modify the lifetime of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+
+Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
+
+```json
+{
+ "lifetime": 3600
+}
+```
+### Stop a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STOPPED"
+}
+```
+
+### Start a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STARTED"
+}
+```
+
+### Update to flex up/down the no of containers (instances) of a component of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
+
+##### PUT Request JSON
+```json
+{
+ "name": "hello",
+ "number_of_containers": 3
+}
+```
+
+### Destroy a service
+DELETE URL - http://localhost:9191/ws/v1/services/hello-world
+
+***
+
+### Create a complicated service - HBase
+POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
+
+##### POST Request JSON
+
+```json
+{
+ "name": "hbase-app-1",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "hbasemaster",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "regionserver",
+ "number_of_containers": 3,
+ "unique_component_support": "true",
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010",
+ "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
+ "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
+ }
+}
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd4490a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 17f8c95..b084be7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -17,18 +17,30 @@
swagger: '2.0'
info:
- title: "[YARN-4793] Simplified API layer for services and beyond"
+ title: "YARN Simplified API layer for services"
description: |
- Bringing a new service on YARN today is not a simple experience. The APIs of existing frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) or writing a complex spec (for declarative frameworks). In addition to building critical building blocks inside YARN (as part of other efforts at link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a need for simplifying the user facing story for building services. Experience of projects like Apache Slider running real-life services like HBase, Storm, Accumulo, Solr etc, gives us some very good insights on how simplified APIs for services should look like.
+ Bringing a new service on YARN today is not a simple experience. The APIs of
+ existing frameworks are either too low level (native YARN), require writing
+ new code (for frameworks with programmatic APIs) or writing a complex spec
+ (for declarative frameworks). In addition to building critical building blocks
+ inside YARN (as part of other efforts at YARN-4692), there is a need for
+ simplifying the user facing story for building services. Experience of projects
+ like Apache Slider running real-life services like HBase, Storm, Accumulo,
+ Solr etc, gives us some very good insights on how simplified APIs for services
+ should look like.
+ To this end, we should look at a new simple-services API layer backed by REST
+ interfaces. This API can be used to create and manage the lifecycle of YARN
+ services. Services here can range from simple single-component service to
+ complex multi-component assemblies needing orchestration. YARN-4793 tracks
+ this effort.
- To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component service to complex multi-component assemblies needing orchestration.
-
-
- We should also look at making this a unified REST based entry point for other important features like resource-profile management (link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), package-definitions' lifecycle-management and service-discovery (link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]). We also need to flesh out its relation to our present much lower level REST APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN for application-submission and management.
-
-
- This document spotlights on this specification. In most of the cases, the application owner will not be forced to make any changes to their application. This is primarily true if the application is packaged with containerization technologies like docker. Irrespective of how complex the application is, there will be hooks provided at appropriate layers to allow pluggable and customizable application behavior.
+ This document spotlights on this specification. In most of the cases, the
+ application owner will not be forced to make any changes to their applications.
+ This is primarily true if the application is packaged with containerization
+ technologies like docker. Irrespective of how complex the application is,
+ there will be hooks provided at appropriate layers to allow pluggable and
+ customizable application behavior.
version: "1.0.0"
license:
@@ -39,7 +51,6 @@ host: host.mycompany.com
# array of all schemes that your API supports
schemes:
- http
- - https
# will be prefixed to all paths
basePath: /ws/v1/
consumes:
@@ -47,9 +58,17 @@ consumes:
produces:
- application/json
paths:
+ /services/version:
+ get:
+ summary: Get current version of the API server.
+ description: Get current version of the API server.
+ responses:
+ 200:
+ description: Successful request
+
/services:
get:
- summary: List of services running in the cluster
+ summary: (TBD) List of services running in the cluster.
description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
responses:
200:
@@ -74,15 +93,62 @@ paths:
$ref: '#/definitions/Service'
responses:
202:
- description: Request accepted
+ description: The request to create a service is accepted
+ 400:
+ description: Invalid service definition provided in the request body
+ 500:
+ description: Failed to create a service
default:
description: Unexpected error
schema:
$ref: '#/definitions/ServiceStatus'
/services/{service_name}:
+ put:
+ summary: Update a service or upgrade the binary version of the components of a running service
+ description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service.
+ The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
+ parameters:
+ - name: service_name
+ in: path
+ description: Service name
+ required: true
+ type: string
+ - name: Service
+ in: body
+ description: The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service
+ required: true
+ schema:
+ $ref: '#/definitions/Service'
+ responses:
+ 204:
+ description: Update or upgrade was successful
+ 404:
+ description: Service does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
+ delete:
+ summary: Destroy a service
+ description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
+ parameters:
+ - name: service_name
+ in: path
+ description: Service name
+ required: true
+ type: string
+ responses:
+ 204:
+ description: Destroy was successful
+ 404:
+ description: Service does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
get:
- summary: Get service details
+ summary: Get details of a service.
description: Return the details (including containers) of a running service
parameters:
- name: service_name
@@ -108,43 +174,36 @@ paths:
description: Unexpected error
schema:
$ref: '#/definitions/ServiceStatus'
+ /services/{service_name}/components/{component_name}:
put:
- summary: Update a service or upgrade the binary version of the components of a running service
- description: Update the runtime properties of a service. As of now, only update of lifetime and number of instances (flexing) of the components of a service is supported. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts.
+ summary: Flex a component's number of instances.
+ description: Set a component's desired number of instanes
parameters:
- name: service_name
in: path
description: Service name
required: true
type: string
- responses:
- 204:
- description: Update or upgrade was successful
- 404:
- description: Service does not exist
- default:
- description: Unexpected error
- schema:
- $ref: '#/definitions/ServiceStatus'
- delete:
- summary: Destroy service
- description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet.
- parameters:
- - name: service_name
+ - name: component_name
in: path
- description: Service name
+ description: Component name
required: true
type: string
+ - name: Component
+ in: body
+ description: The definition of a component which contains the updated number of instances.
+ required: true
+ schema:
+ $ref: '#/definitions/Component'
responses:
- 204:
- description: Destroy was successful
+ 200:
+ description: Flex was successful
404:
description: Service does not exist
default:
description: Unexpected error
schema:
$ref: '#/definitions/ServiceStatus'
-
definitions:
Service:
description: a service resource has the following attributes.
@@ -183,7 +242,7 @@ definitions:
format: int64
description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
placement_policy:
- description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
+ description: (TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
$ref: '#/definitions/PlacementPolicy'
components:
description: Components of a service.
@@ -420,4 +479,3 @@ definitions:
type: integer
format: int32
description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd4490a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
new file mode 100644
index 0000000..f56139a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
@@ -0,0 +1,606 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN Simplified API layer for services
+
+## Overview
+Bringing a new service on YARN today is not a simple experience. The APIs of
+existing frameworks are either too low level (native YARN), require writing
+new code (for frameworks with programmatic APIs) or writing a complex spec
+(for declarative frameworks). In addition to building critical building blocks
+inside YARN (as part of other efforts at
+[YARN-4692](https://issues.apache.org/jira/browse/YARN-4692)), there is a need for
+simplifying the user facing story for building services. Experience of projects
+like Apache Slider running real-life services like HBase, Storm, Accumulo,
+Solr etc, gives us some very good insights on how simplified APIs for services
+should look like.
+
+To this end, we should look at a new simple-services API layer backed by REST
+interfaces. This API can be used to create and manage the lifecycle of YARN
+services. Services here can range from simple single-component service to
+complex multi-component assemblies needing orchestration.
+[YARN-4793](https://issues.apache.org/jira/browse/YARN-4793) tracks this
+effort.
+
+This document spotlights on this specification. In most of the cases, the
+application owner will not be forced to make any changes to their applications.
+This is primarily true if the application is packaged with containerization
+technologies like docker. Irrespective of how complex the application is,
+there will be hooks provided at appropriate layers to allow pluggable and
+customizable application behavior.
+
+
+### Version information
+Version: 1.0.0
+
+### License information
+License: Apache 2.0
+License URL: http://www.apache.org/licenses/LICENSE-2.0.html
+
+### URI scheme
+Host: host.mycompany.com
+
+BasePath: /ws/v1/
+
+Schemes: HTTP
+
+### Consumes
+
+* application/json
+
+
+### Produces
+
+* application/json
+
+
+## Paths
+### Create a service
+```
+POST /services
+```
+
+#### Description
+
+Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|BodyParameter|Service|Service request object|true|Service||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|202|The request to create a service is accepted|No Content|
+|400|Invalid service definition provided in the request body|No Content|
+|500|Failed to create a service|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### (TBD) List of services running in the cluster.
+```
+GET /services
+```
+
+#### Description
+
+Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|An array of services|Service array|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get current version of the API server.
+```
+GET /services/version
+```
+
+#### Description
+
+Get current version of the API server.
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|Successful request|No Content|
+
+
+### Update a service or upgrade the binary version of the components of a running service
+```
+PUT /services/{service_name}
+```
+
+#### Description
+
+Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+|BodyParameter|Service|The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service|true|Service||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|204|Update or upgrade was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Destroy a service
+```
+DELETE /services/{service_name}
+```
+
+#### Description
+
+Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|204|Destroy was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get details of a service.
+```
+GET /services/{service_name}
+```
+
+#### Description
+
+Return the details (including containers) of a running service
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|a service object|object|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Flex a component's number of instances.
+```
+PUT /services/{service_name}/components/{component_name}
+```
+
+#### Description
+
+Set a component's desired number of instanes
+
+#### Parameters
+|Type|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|----|
+|PathParameter|service_name|Service name|true|string||
+|PathParameter|component_name|Component name|true|string||
+|BodyParameter|Component|The definition of a component which contains the updated number of instances.|true|Component||
+
+
+#### Responses
+|HTTP Code|Description|Schema|
+|----|----|----|
+|200|Flex was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+## Definitions
+### Artifact
+
+Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|id|Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.|true|string||
+|type|Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).|false|enum (DOCKER, TARBALL, SERVICE)|DOCKER|
+|uri|Artifact location to support multiple artifact stores (optional).|false|string||
+
+
+### Component
+
+One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|name|Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.|true|string||
+|dependencies|An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.|false|string array||
+|readiness_check|Readiness check for this component.|false|ReadinessCheck||
+|artifact|Artifact of the component (optional). If not specified, the service level global artifact takes effect.|false|Artifact||
+|launch_command|The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).|false|string||
+|resource|Resource of this component (optional). If not specified, the service level global resource takes effect.|false|Resource||
+|number_of_containers|Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.|false|integer (int64)||
+|run_privileged_container|Run all containers of this component in privileged mode (YARN-4262).|false|boolean||
+|placement_policy|Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.|false|PlacementPolicy||
+|configuration|Config properties for this component.|false|Configuration||
+|quicklinks|A list of quicklink keys defined at the service level, and to be resolved by this component.|false|string array||
+
+
+### ConfigFile
+
+A config file that needs to be created and made available as a volume in a service component container.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|type|Config file in the standard format like xml, properties, json, yaml, template.|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, ENV, HADOOP_XML)||
+|dest_file|The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.|false|string||
+|src_file|This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.|false|string||
+|props|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
+
+
+### Configuration
+
+Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|properties|A blob of key-value pairs of common service properties.|false|object||
+|env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object||
+|files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array||
+
+
+### Container
+
+An instance of a running service container.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|id|Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.|false|string||
+|launch_time|The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.|false|string (date)||
+|ip|IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
+|hostname|Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.|false|string||
+|bare_host|The bare node or host in which the container is running, e.g. cn008.example.com.|false|string||
+|state|State of the container of a service.|false|ContainerState||
+|component_name|Name of the component that this container instance belongs to.|false|string||
+|resource|Resource used for this container.|false|Resource||
+|artifact|Artifact used for this container.|false|Artifact||
+|privileged_container|Container running in privileged mode or not.|false|boolean||
+
+
+### ContainerState
+
+The current state of the container of a service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|state|enum of the state of the container|false|enum (INIT, STARTED, READY)||
+
+
+### PlacementPolicy
+
+Placement policy of an instance of a service. This feature is in the works in YARN-6592.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|label|Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.|false|string||
+
+
+### ReadinessCheck
+
+A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|type|E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).|true|enum (HTTP, PORT)||
+|props|A blob of key value pairs that will be used to configure the check.|false|object||
+|artifact|Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET|false|Artifact||
+
+
+### Resource
+
+Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string||
+|cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)||
+|memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string||
+
+
+### Service
+
+a service resource has the following attributes.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|name|A unique service name. If Registry DNS is enabled, the max length is 63 characters.|true|string||
+|id|A unique service id.|false|string||
+|artifact|Artifact of single-component service.|false|Artifact||
+|resource|Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.|false|Resource||
+|launch_command|The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.|false|string||
+|launch_time|The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.|false|string (date)||
+|number_of_containers|Number of containers for each component in the service. Each component can further override this service-level global default.|false|integer (int64)||
+|number_of_running_containers|In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.|false|integer (int64)||
+|lifetime|Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.|false|integer (int64)||
+|placement_policy|(TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.|false|PlacementPolicy||
+|components|Components of a service.|false|Component array||
+|configuration|Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.|false|Configuration||
+|containers|Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.|false|Container array||
+|state|State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.|false|ServiceState||
+|quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
+|queue|The YARN queue that this service should be submitted to.|false|string||
+
+
+### ServiceState
+
+The current state of a service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, READY, STOPPED, FAILED)||
+
+
+### ServiceStatus
+
+The current status of a submitted service, returned as a response to the GET API.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|diagnostics|Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.|false|string||
+|state|Service state.|false|ServiceState||
+|code|An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.|false|integer (int32)||
+
+
+
+## Examples
+
+### Create a simple single-component service with most attribute values as defaults
+POST URL - http://localhost:9191/ws/v1/services
+
+##### POST Request JSON
+```json
+{
+ "name": "hello-world",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "nginx:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "./start_nginx.sh",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+
+##### GET Response JSON
+GET URL - http://localhost:9191/ws/v1/services/hello-world
+
+Note, lifetime value of -1 means unlimited lifetime.
+
+```json
+{
+ "name": "hello-world",
+ "id": "application_1503963985568_0002",
+ "lifetime": -1,
+ "components": [
+ {
+ "name": "hello",
+ "dependencies": [],
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": [],
+ "containers": [
+ {
+ "id": "container_e03_1503963985568_0002_01_000001",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051512412,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-0"
+ },
+ {
+ "id": "container_e03_1503963985568_0002_01_000002",
+ "ip": "10.22.8.143",
+ "hostname": "myhost.local",
+ "state": "READY",
+ "launch_time": 1504051536450,
+ "bare_host": "10.22.8.143",
+ "component_name": "hello-1"
+ }
+ ],
+ "launch_command": "./start_nginx.sh",
+ "number_of_containers": 1,
+ "run_privileged_container": false
+ }
+ ],
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": {}
+}
+
+```
+### Update to modify the lifetime of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+
+Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
+
+```json
+{
+ "lifetime": 3600
+}
+```
+### Stop a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STOPPED"
+}
+```
+
+### Start a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STARTED"
+}
+```
+
+### Update to flex up/down the no of containers (instances) of a component of a service
+PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
+
+##### PUT Request JSON
+```json
+{
+ "name": "hello",
+ "number_of_containers": 3
+}
+```
+
+### Destroy a service
+DELETE URL - http://localhost:9191/ws/v1/services/hello-world
+
+***
+
+### Create a complicated service - HBase
+POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
+
+##### POST Request JSON
+
+```json
+{
+ "name": "hbase-app-1",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "hbasemaster",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "regionserver",
+ "number_of_containers": 3,
+ "unique_component_support": "true",
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "props": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "props": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010",
+ "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
+ "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
+ }
+}
+```
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/58] [abbrv] hadoop git commit: Rebase to trunk
Posted by ji...@apache.org.
Rebase to trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c30b1a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c30b1a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c30b1a9
Branch: refs/heads/yarn-native-services
Commit: 3c30b1a97d4d93a5714a91db90749c2a581bfdc4
Parents: 9273480
Author: Jian He <ji...@apache.org>
Authored: Fri Oct 13 11:15:02 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:18 2017 -0800
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c30b1a9/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 727384c..cc269d8 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -51,7 +51,6 @@ function hadoop_usage
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "service" client "deploy/manage a service on YARN"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
new file mode 100644
index 0000000..684f655
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Formatter;
+import java.util.Locale;
+
+/**
+ * Various utils to work with the monitor
+ */
+public final class MonitorUtils {
+ protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
+ .class);
+
+ private MonitorUtils() {
+ }
+
+ public static String toPlural(int val) {
+ return val != 1 ? "s" : "";
+ }
+
+ /**
+ * Convert milliseconds to human time -the exact format is unspecified
+ * @param milliseconds a time in milliseconds
+ * @return a time that is converted to human intervals
+ */
+ public static String millisToHumanTime(long milliseconds) {
+ StringBuilder sb = new StringBuilder();
+ // Send all output to the Appendable object sb
+ Formatter formatter = new Formatter(sb, Locale.US);
+
+ long s = Math.abs(milliseconds / 1000);
+ long m = Math.abs(milliseconds % 1000);
+ if (milliseconds > 0) {
+ formatter.format("%d.%03ds", s, m);
+ } else if (milliseconds == 0) {
+ formatter.format("0");
+ } else {
+ formatter.format("-%d.%03ds", s, m);
+ }
+ return sb.toString();
+ }
+
+ public static Probe getProbe(ReadinessCheck readinessCheck) {
+ if (readinessCheck == null) {
+ return null;
+ }
+ if (readinessCheck.getType() == null) {
+ return null;
+ }
+ try {
+ switch (readinessCheck.getType()) {
+ case HTTP:
+ return HttpProbe.create(readinessCheck.getProps());
+ case PORT:
+ return PortProbe.create(readinessCheck.getProps());
+ default:
+ return null;
+ }
+ } catch (Throwable t) {
+ throw new IllegalArgumentException("Error creating readiness check " +
+ t);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
new file mode 100644
index 0000000..aba5859
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.Map;
+
+/**
+ * Probe for a port being open.
+ */
+public class PortProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
+ private final int port;
+ private final int timeout;
+
+ public PortProbe(int port, int timeout) {
+ super("Port probe of " + port + " for " + timeout + "ms", null);
+ this.port = port;
+ this.timeout = timeout;
+ }
+
+ public static PortProbe create(Map<String, String> props)
+ throws IOException {
+ int port = getPropertyInt(props, PORT_PROBE_PORT, null);
+
+ if (port >= 65536) {
+ throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
+ "range");
+ }
+
+ int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
+ PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
+
+ return new PortProbe(port, timeout);
+ }
+
+ /**
+ * Try to connect to the (host,port); a failure to connect within
+ * the specified timeout is a failure.
+ * @param instance role instance
+ * @return the outcome
+ */
+ @Override
+ public ProbeStatus ping(ComponentInstance instance) {
+ ProbeStatus status = new ProbeStatus();
+
+ if (instance.getContainerStatus() == null || SliderUtils
+ .isEmpty(instance.getContainerStatus().getIPs())) {
+ status.fail(this, new IOException(
+ instance.getCompInstanceName() + ": IP is not available yet"));
+ return status;
+ }
+
+ String ip = instance.getContainerStatus().getIPs().get(0);
+ InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
+ Socket socket = new Socket();
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
+ .toString() + ", timeout=" + MonitorUtils
+ .millisToHumanTime(timeout));
+ }
+ socket.connect(sockAddr, timeout);
+ status.succeed(this);
+ } catch (Throwable e) {
+ String error =
+ instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
+ log.debug(error, e);
+ status.fail(this, new IOException(error, e));
+ } finally {
+ IOUtils.closeSocket(socket);
+ }
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
new file mode 100644
index 0000000..3237a2b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Base class of all probes.
+ */
+public abstract class Probe implements MonitorKeys {
+
+ protected final Configuration conf;
+ private String name;
+
+ /**
+ * Create a probe of a specific name
+ *
+ * @param name probe name
+ * @param conf configuration being stored.
+ */
+ public Probe(String name, Configuration conf) {
+ this.name = name;
+ this.conf = conf;
+ }
+
+
+ protected void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+
+ @Override
+ public String toString() {
+ return getName();
+ }
+
+ public static String getProperty(Map<String, String> props, String name,
+ String defaultValue) throws IOException {
+ String value = props.get(name);
+ if (StringUtils.isEmpty(value)) {
+ if (defaultValue == null) {
+ throw new IOException(name + " not specified");
+ }
+ return defaultValue;
+ }
+ return value;
+ }
+
+ public static int getPropertyInt(Map<String, String> props, String name,
+ Integer defaultValue) throws IOException {
+ String value = props.get(name);
+ if (StringUtils.isEmpty(value)) {
+ if (defaultValue == null) {
+ throw new IOException(name + " not specified");
+ }
+ return defaultValue;
+ }
+ return Integer.parseInt(value);
+ }
+
+ /**
+ * perform any prelaunch initialization
+ */
+ public void init() throws IOException {
+
+ }
+
+ /**
+ * Ping the endpoint. All exceptions must be caught and included in the
+ * (failure) status.
+ *
+ * @param instance instance to ping
+ * @return the status
+ */
+ public abstract ProbeStatus ping(ComponentInstance instance);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
new file mode 100644
index 0000000..bc62dcd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * Status message of a probe. This is designed to be sent over the wire, though the exception
+ * Had better be unserializable at the far end if that is to work.
+ */
+public final class ProbeStatus implements Serializable {
+ private static final long serialVersionUID = 165468L;
+
+ private long timestamp;
+ private String timestampText;
+ private boolean success;
+ private boolean realOutcome;
+ private String message;
+ private Throwable thrown;
+ private transient Probe originator;
+
+ public ProbeStatus() {
+ }
+
+ public ProbeStatus(long timestamp, String message, Throwable thrown) {
+ this.success = false;
+ this.message = message;
+ this.thrown = thrown;
+ setTimestamp(timestamp);
+ }
+
+ public ProbeStatus(long timestamp, String message) {
+ this.success = true;
+ setTimestamp(timestamp);
+ this.message = message;
+ this.thrown = null;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ timestampText = new Date(timestamp).toString();
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ /**
+ * Set both the success and the real outcome bits to the same value
+ * @param success the new value
+ */
+ public void setSuccess(boolean success) {
+ this.success = success;
+ realOutcome = success;
+ }
+
+ public String getTimestampText() {
+ return timestampText;
+ }
+
+ public boolean getRealOutcome() {
+ return realOutcome;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public Throwable getThrown() {
+ return thrown;
+ }
+
+ public void setThrown(Throwable thrown) {
+ this.thrown = thrown;
+ }
+
+ /**
+ * Get the probe that generated this result. May be null
+ * @return a possibly null reference to a probe
+ */
+ public Probe getOriginator() {
+ return originator;
+ }
+
+ /**
+ * The probe has succeeded -capture the current timestamp, set
+ * success to true, and record any other data needed.
+ * @param probe probe
+ */
+ public void succeed(Probe probe) {
+ finish(probe, true, probe.getName(), null);
+ }
+
+ /**
+ * A probe has failed either because the test returned false, or an exception
+ * was thrown. The {@link #success} field is set to false, any exception
+ * thrown is recorded.
+ * @param probe probe that failed
+ * @param thrown an exception that was thrown.
+ */
+ public void fail(Probe probe, Throwable thrown) {
+ finish(probe, false, "Failure in " + probe, thrown);
+ }
+
+ public void finish(Probe probe, boolean succeeded, String text, Throwable thrown) {
+ setTimestamp(System.currentTimeMillis());
+ setSuccess(succeeded);
+ originator = probe;
+ message = text;
+ this.thrown = thrown;
+ }
+
+ @Override
+ public String toString() {
+ LogEntryBuilder builder = new LogEntryBuilder("Probe Status");
+ builder.elt("time", timestampText)
+ .elt("outcome", (success ? "success" : "failure"));
+
+ if (success != realOutcome) {
+ builder.elt("originaloutcome", (realOutcome ? "success" : "failure"));
+ }
+ builder.elt("message", message);
+ if (thrown != null) {
+ builder.elt("exception", thrown);
+ }
+
+ return builder.toString();
+ }
+
+ /**
+ * Flip the success bit on while the real outcome bit is kept false
+ */
+ public void markAsSuccessful() {
+ success = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
new file mode 100644
index 0000000..0d11be2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public abstract class AbstractClientProvider {
+
+ public AbstractClientProvider() {
+ }
+
+ /**
+ * Generates a fixed format of application tags given one or more of
+ * application name, version and description. This allows subsequent query for
+ * an application with a name only, version only or description only or any
+ * combination of those as filters.
+ *
+ * @param appName name of the application
+ * @param appVersion version of the application
+ * @param appDescription brief description of the application
+ * @return
+ */
+ public static final Set<String> createApplicationTags(String appName,
+ String appVersion, String appDescription) {
+ Set<String> tags = new HashSet<>();
+ tags.add(SliderUtils.createNameTag(appName));
+ if (appVersion != null) {
+ tags.add(SliderUtils.createVersionTag(appVersion));
+ }
+ if (appDescription != null) {
+ tags.add(SliderUtils.createDescriptionTag(appDescription));
+ }
+ return tags;
+ }
+
+ /**
+ * Validate the artifact.
+ * @param artifact
+ */
+ public abstract void validateArtifact(Artifact artifact, FileSystem
+ fileSystem) throws IOException;
+
+ protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException;
+
+ /**
+ * Validate the config files.
+ * @param configFiles config file list
+ * @param fs file system
+ */
+ public void validateConfigFiles(List<ConfigFile> configFiles,
+ FileSystem fs) throws IOException {
+ Set<String> destFileSet = new HashSet<>();
+
+ for (ConfigFile file : configFiles) {
+ if (file.getType() == null) {
+ throw new IllegalArgumentException("File type is empty");
+ }
+
+ if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
+ .isEmpty(file.getSrcFile())) {
+ throw new IllegalArgumentException(
+ "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
+
+ }
+ if (!StringUtils.isEmpty(file.getSrcFile())) {
+ Path p = new Path(file.getSrcFile());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException(
+ "Src_file does not exist for config file: " + file
+ .getSrcFile());
+ }
+ }
+
+ if (StringUtils.isEmpty(file.getDestFile())) {
+ throw new IllegalArgumentException("Dest_file is empty.");
+ }
+
+ if (destFileSet.contains(file.getDestFile())) {
+ throw new IllegalArgumentException(
+ "Duplicated ConfigFile exists: " + file.getDestFile());
+ }
+ destFileSet.add(file.getDestFile());
+
+ java.nio.file.Path destPath = Paths.get(file.getDestFile());
+ if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
+ throw new IllegalArgumentException("Non-absolute dest_file has more " +
+ "than one path element");
+ }
+
+ // provider-specific validation
+ validateConfigFile(file, fs);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
new file mode 100644
index 0000000..8d607ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_INTERVAL;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_MAX;
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+
+public abstract class AbstractProviderService implements ProviderService,
+ YarnServiceConstants {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(AbstractProviderService.class);
+
+ public abstract void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Service service)
+ throws IOException;
+
+ public void buildContainerLaunchContext(AbstractLauncher launcher,
+ Service service, ComponentInstance instance,
+ SliderFileSystem fileSystem, Configuration yarnConf)
+ throws IOException, SliderException {
+ Component component = instance.getComponent().getComponentSpec();;
+ processArtifact(launcher, instance, fileSystem, service);
+
+ ServiceContext context =
+ instance.getComponent().getScheduler().getContext();
+ // Generate tokens (key-value pair) for config substitution.
+ // Get pre-defined tokens
+ Map<String, String> globalTokens =
+ instance.getComponent().getScheduler().globalTokens;
+ Map<String, String> tokensForSubstitution = ProviderUtils
+ .initCompTokensForSubstitute(instance);
+ tokensForSubstitution.putAll(globalTokens);
+ // Set the environment variables in launcher
+ launcher.putEnv(SliderUtils
+ .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
+ launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
+ launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ if (System.getenv(HADOOP_USER_NAME) != null) {
+ launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
+ }
+ launcher.setEnv("LANG", "en_US.UTF-8");
+ launcher.setEnv("LC_ALL", "en_US.UTF-8");
+ launcher.setEnv("LANGUAGE", "en_US.UTF-8");
+
+ for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
+ tokensForSubstitution.put($(entry.getKey()), entry.getValue());
+ }
+ //TODO add component host tokens?
+// ProviderUtils.addComponentHostTokens(tokensForSubstitution, amState);
+
+ // create config file on hdfs and add local resource
+ ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
+ component, tokensForSubstitution, instance, context);
+
+ // substitute launch command
+ String launchCommand = ProviderUtils
+ .substituteStrWithTokens(component.getLaunchCommand(),
+ tokensForSubstitution);
+ CommandLineBuilder operation = new CommandLineBuilder();
+ operation.add(launchCommand);
+ operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+ launcher.addCommand(operation.build());
+
+ // By default retry forever every 30 seconds
+ launcher.setRetryContext(YarnServiceConf
+ .getInt(CONTAINER_RETRY_MAX, -1, service.getConfiguration(),
+ yarnConf), YarnServiceConf
+ .getInt(CONTAINER_RETRY_INTERVAL, 30000, service.getConfiguration(),
+ yarnConf));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
new file mode 100644
index 0000000..0f949e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for factories.
+ */
+public abstract class ProviderFactory {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ProviderFactory.class);
+
+ protected ProviderFactory() {}
+
+ public abstract AbstractClientProvider createClientProvider();
+
+ public abstract ProviderService createServerProvider();
+
+ public static synchronized ProviderService getProviderService(Artifact
+ artifact) {
+ return createServiceProviderFactory(artifact).createServerProvider();
+ }
+
+ public static synchronized AbstractClientProvider getClientProvider(Artifact
+ artifact) {
+ return createServiceProviderFactory(artifact).createClientProvider();
+ }
+
+ /**
+ * Create a provider for a specific service
+ * @param artifact artifact
+ * @return provider factory
+ */
+ public static synchronized ProviderFactory createServiceProviderFactory(
+ Artifact artifact) {
+ if (artifact == null || artifact.getType() == null) {
+ LOG.debug("Loading service provider type default");
+ return DefaultProviderFactory.getInstance();
+ }
+ LOG.debug("Loading service provider type {}", artifact.getType());
+ switch (artifact.getType()) {
+ // TODO add handling for custom types?
+ // TODO handle service
+ case DOCKER:
+ return DockerProviderFactory.getInstance();
+ case TARBALL:
+ return TarballProviderFactory.getInstance();
+ default:
+ throw new IllegalArgumentException(String.format("Resolution error, " +
+ "%s should not be passed to createServiceProviderFactory",
+ artifact.getType()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
new file mode 100644
index 0000000..eb721b4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+import java.io.IOException;
+
+public interface ProviderService {
+
+ /**
+ * Set up the entire container launch context
+ */
+ void buildContainerLaunchContext(AbstractLauncher containerLauncher,
+ Service service, ComponentInstance instance,
+ SliderFileSystem sliderFileSystem, Configuration yarnConf)
+ throws IOException, SliderException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
new file mode 100644
index 0000000..ec0c2ca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
+import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
+
+/**
+ * This is a factoring out of methods handy for providers. It's bonded to a log
+ * at construction time.
+ */
+public class ProviderUtils implements YarnServiceConstants {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(ProviderUtils.class);
+
+
+ /**
+ * Add oneself to the classpath. This does not work
+ * on minicluster test runs where the JAR is not built up.
+ * @param providerResources map of provider resources to add these entries to
+ * @param providerClass provider to add
+ * @param jarName name of the jar to use
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libdir relative directory to place resources
+ * @param miniClusterTestRun true if minicluster is being used
+ * @return true if the class was found in a JAR
+ *
+ * @throws FileNotFoundException if the JAR was not found and this is NOT
+ * a mini cluster test run
+ * @throws IOException IO problems
+ * @throws SliderException any Slider problem
+ */
+ public static boolean addProviderJar(
+ Map<String, LocalResource> providerResources,
+ Class providerClass,
+ String jarName,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libdir,
+ boolean miniClusterTestRun) throws
+ IOException,
+ SliderException {
+ try {
+ SliderUtils.putJar(providerResources,
+ sliderFileSystem,
+ providerClass,
+ tempPath,
+ libdir,
+ jarName);
+ return true;
+ } catch (FileNotFoundException e) {
+ if (miniClusterTestRun) {
+ return false;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Loads all dependency jars from the default path.
+ * @param providerResources map of provider resources to add these entries to
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libDir relative directory to place resources
+ * @param libLocalSrcDir explicitly supplied local libs dir
+ * @throws IOException trouble copying to HDFS
+ * @throws SliderException trouble copying to HDFS
+ */
+ public static void addAllDependencyJars(
+ Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String libLocalSrcDir)
+ throws IOException, SliderException {
+ if (SliderUtils.isSet(libLocalSrcDir)) {
+ File file = new File(libLocalSrcDir);
+ if (!file.exists() || !file.isDirectory()) {
+ throw new BadCommandArgumentsException(
+ "Supplied lib src dir %s is not valid", libLocalSrcDir);
+ }
+ }
+ SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
+ libDir, libLocalSrcDir);
+ }
+
+ public static String substituteStrWithTokens(String content,
+ Map<String, String> tokensForSubstitution) {
+ for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+ content =
+ content.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+ }
+ return content;
+ }
+
+ // configs will be substituted by corresponding env in tokenMap
+ public static void substituteMapWithTokens(Map<String, String> configs,
+ Map<String, String> tokenMap) {
+ for (Map.Entry<String, String> entry : configs.entrySet()) {
+ String value = entry.getValue();
+ if (tokenMap != null) {
+ for (Map.Entry<String, String> token : tokenMap.entrySet()) {
+ value =
+ value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+ }
+ }
+ entry.setValue(value);
+ }
+ }
+
+ /**
+ * Localize the service keytabs for the service.
+ * @param launcher container launcher
+ * @param fileSystem file system
+ * @throws IOException trouble uploading to HDFS
+ */
+ public void localizeServiceKeytabs(AbstractLauncher launcher,
+ SliderFileSystem fileSystem, Service service) throws IOException {
+
+ Configuration conf = service.getConfiguration();
+ String keytabPathOnHost =
+ conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
+ if (SliderUtils.isUnset(keytabPathOnHost)) {
+ String amKeytabName =
+ conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir =
+ conf.getProperty(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
+ // we need to localize the keytab files in the directory
+ Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
+ service.getName());
+ boolean serviceKeytabsDeployed = false;
+ if (fileSystem.getFileSystem().exists(keytabDirPath)) {
+ FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
+ keytabDirPath);
+ LocalResource keytabRes;
+ for (FileStatus keytab : keytabs) {
+ if (!amKeytabName.equals(keytab.getPath().getName())
+ && keytab.getPath().getName().endsWith(".keytab")) {
+ serviceKeytabsDeployed = true;
+ log.info("Localizing keytab {}", keytab.getPath().getName());
+ keytabRes = fileSystem.createAmResource(keytab.getPath(),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(KEYTAB_DIR + "/" +
+ keytab.getPath().getName(),
+ keytabRes);
+ }
+ }
+ }
+ if (!serviceKeytabsDeployed) {
+ log.warn("No service keytabs for the service have been localized. "
+ + "If the service requires keytabs for secure operation, "
+ + "please ensure that the required keytabs have been uploaded "
+ + "to the folder {}", keytabDirPath);
+ }
+ }
+ }
+
+ // 1. Create all config files for a component on hdfs for localization
+ // 2. Add the config file to localResource
+ public static synchronized void createConfigFileAndAddLocalResource(
+ AbstractLauncher launcher, SliderFileSystem fs, Component component,
+ Map<String, String> tokensForSubstitution, ComponentInstance instance,
+ ServiceContext context) throws IOException {
+ Path compDir =
+ new Path(new Path(fs.getAppDir(), "components"), component.getName());
+ Path compInstanceDir =
+ new Path(compDir, instance.getCompInstanceName());
+ if (!fs.getFileSystem().exists(compInstanceDir)) {
+ log.info(instance.getCompInstanceId() + ": Creating dir on hdfs: " + compInstanceDir);
+ fs.getFileSystem().mkdirs(compInstanceDir,
+ new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ instance.setCompInstanceDir(compInstanceDir);
+ } else {
+ log.info("Component instance conf dir already exists: " + compInstanceDir);
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("Tokens substitution for component instance: " + instance
+ .getCompInstanceName() + System.lineSeparator()
+ + tokensForSubstitution);
+ }
+
+ for (ConfigFile originalFile : component.getConfiguration().getFiles()) {
+ ConfigFile configFile = originalFile.copy();
+ String fileName = new Path(configFile.getDestFile()).getName();
+
+ // substitute file name
+ for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+ configFile.setDestFile(configFile.getDestFile()
+ .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
+ }
+
+ Path remoteFile = new Path(compInstanceDir, fileName);
+ if (!fs.getFileSystem().exists(remoteFile)) {
+ log.info("Saving config file on hdfs for component " + instance
+ .getCompInstanceName() + ": " + configFile);
+
+ if (configFile.getSrcFile() != null) {
+ // Load config file template
+ switch (configFile.getType()) {
+ case HADOOP_XML:
+ // Hadoop_xml_template
+ resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(),
+ tokensForSubstitution, configFile, remoteFile, context);
+ break;
+ case TEMPLATE:
+ // plain-template
+ resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(),
+ tokensForSubstitution, configFile, remoteFile, context);
+ break;
+ default:
+ log.info("Not supporting loading src_file for " + configFile);
+ break;
+ }
+ } else {
+ // non-template
+ resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
+ instance, configFile, fileName, remoteFile);
+ }
+ }
+
+ // Add resource for localization
+ LocalResource configResource =
+ fs.createAmResource(remoteFile, LocalResourceType.FILE);
+ File destFile = new File(configFile.getDestFile());
+ String symlink = APP_CONF_DIR + "/" + fileName;
+ if (destFile.isAbsolute()) {
+ launcher.addLocalResource(symlink, configResource,
+ configFile.getDestFile());
+ log.info("Add config file for localization: " + symlink + " -> "
+ + configResource.getResource().getFile() + ", dest mount path: "
+ + configFile.getDestFile());
+ } else {
+ launcher.addLocalResource(symlink, configResource);
+ log.info("Add config file for localization: " + symlink + " -> "
+ + configResource.getResource().getFile());
+ }
+ }
+ }
+
+ private static void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
+ Map<String, String> tokensForSubstitution, ComponentInstance instance,
+ ConfigFile configFile, String fileName, Path remoteFile)
+ throws IOException {
+ // substitute non-template configs
+ substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
+
+ // write configs onto hdfs
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(fileName,
+ configFile.getProps().entrySet());
+ if (!fs.getFileSystem().exists(remoteFile)) {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(
+ ConfigFormat.resolve(configFile.getType().toString()),
+ publishedConfiguration);
+ try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) {
+ configurationOutputter.save(os);
+ os.flush();
+ }
+ } else {
+ log.info("Component instance = " + instance.getCompInstanceName()
+ + ", config file already exists: " + remoteFile);
+ }
+ }
+
+ // 1. substitute config template - only handle hadoop_xml format
+ // 2. save on hdfs
+ @SuppressWarnings("unchecked")
+ private static void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs,
+ Map<String, String> tokensForSubstitution, ConfigFile configFile,
+ Path remoteFile, ServiceContext context) throws IOException {
+ Map<String, String> conf;
+ try {
+ conf = (Map<String, String>) context.configCache.get(configFile);
+ } catch (ExecutionException e) {
+ log.info("Failed to load config file: " + configFile, e);
+ return;
+ }
+ // make a copy for substitution
+ org.apache.hadoop.conf.Configuration confCopy =
+ new org.apache.hadoop.conf.Configuration(false);
+ for (Map.Entry<String, String> entry : conf.entrySet()) {
+ confCopy.set(entry.getKey(), entry.getValue());
+ }
+ // substitute properties
+ for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
+ confCopy.set(entry.getKey(), entry.getValue());
+ }
+ // substitute env variables
+ for (Map.Entry<String, String> entry : confCopy) {
+ String val = entry.getValue();
+ if (val != null) {
+ for (Map.Entry<String, String> token : tokensForSubstitution
+ .entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+ confCopy.set(entry.getKey(), val);
+ }
+ }
+ }
+ // save on hdfs
+ try (OutputStream output = fs.create(remoteFile)) {
+ confCopy.writeXml(output);
+ log.info("Reading config from: " + configFile.getSrcFile()
+ + ", writing to: " + remoteFile);
+ }
+ }
+
+ // 1) read the template as a string
+ // 2) do token substitution
+ // 3) save on hdfs
+ private static void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
+ Map<String, String> tokensForSubstitution, ConfigFile configFile,
+ Path remoteFile, ServiceContext context) {
+ String content;
+ try {
+ content = (String) context.configCache.get(configFile);
+ } catch (ExecutionException e) {
+ log.info("Failed to load config file: " + configFile, e);
+ return;
+ }
+ // substitute tokens
+ content = substituteStrWithTokens(content, tokensForSubstitution);
+
+ try (OutputStream output = fs.create(remoteFile)) {
+ org.apache.commons.io.IOUtils.write(content, output);
+ } catch (IOException e) {
+ log.info("Failed to create " + remoteFile);
+ }
+ }
+
+ /**
+ * Get initial component token map to be substituted into config values.
+ * @return tokens to replace
+ */
+ public static Map<String, String> initCompTokensForSubstitute(
+ ComponentInstance instance) {
+ Map<String, String> tokens = new HashMap<>();
+ tokens.put(COMPONENT_NAME, instance.getCompSpec().getName());
+ tokens
+ .put(COMPONENT_NAME_LC, instance.getCompSpec().getName().toLowerCase());
+ tokens.put(COMPONENT_INSTANCE_NAME, instance.getCompInstanceName());
+ tokens.put(CONTAINER_ID, instance.getContainer().getId().toString());
+ tokens.put(COMPONENT_ID,
+ String.valueOf(instance.getCompInstanceId().getId()));
+ return tokens;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
new file mode 100644
index 0000000..0920a9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class DefaultClientProvider extends AbstractClientProvider {
+
+ public DefaultClientProvider() {
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ // validate dest_file is not absolute
+ if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Dest_file must not be absolute path: " + configFile.getDestFile());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
new file mode 100644
index 0000000..868bba8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public final class DefaultProviderFactory extends ProviderFactory {
+ private static final ProviderFactory FACTORY = new
+ DefaultProviderFactory();
+
+ private DefaultProviderFactory() {}
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new DefaultProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static ProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
new file mode 100644
index 0000000..a3a0c1f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+
+public class DefaultProviderService extends AbstractProviderService {
+
+ @Override
+ public void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Service service)
+ throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
new file mode 100644
index 0000000..d4a2254
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+
+import java.io.IOException;
+
+public class DockerClientProvider extends AbstractClientProvider
+ implements YarnServiceConstants {
+
+ public DockerClientProvider() {
+ super();
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+ if (artifact == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(artifact.getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
new file mode 100644
index 0000000..f30c002
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+public interface DockerKeys {
+ String PROVIDER_DOCKER = "docker";
+ String DOCKER_PREFIX = "docker.";
+ String DOCKER_IMAGE = DOCKER_PREFIX + "image";
+ String DOCKER_NETWORK = DOCKER_PREFIX + "network";
+ String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged";
+ String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
+
+ String DEFAULT_DOCKER_NETWORK = "bridge";
+ Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
new file mode 100644
index 0000000..57330ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public class DockerProviderFactory extends ProviderFactory {
+ private static final ProviderFactory FACTORY = new
+ DockerProviderFactory();
+
+ private DockerProviderFactory() {
+ }
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new DockerClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new DockerProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static ProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
new file mode 100644
index 0000000..0741947
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+
+public class DockerProviderService extends AbstractProviderService
+ implements DockerKeys {
+
+ public void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Service service) throws IOException{
+ launcher.setYarnDockerMode(true);
+ launcher.setDockerImage(compInstance.getCompSpec().getArtifact().getId());
+ launcher.setDockerNetwork(compInstance.getCompSpec().getConfiguration()
+ .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
+ String domain = compInstance.getComponent().getScheduler().getConfig()
+ .get(RegistryConstants.KEY_DNS_DOMAIN);
+ String hostname;
+ if (domain == null || domain.isEmpty()) {
+ hostname = MessageFormat
+ .format("{0}.{1}.{2}", compInstance.getCompInstanceName(),
+ service.getName(), RegistryUtils.currentUser());
+ } else {
+ hostname = MessageFormat
+ .format("{0}.{1}.{2}.{3}", compInstance.getCompInstanceName(),
+ service.getName(), RegistryUtils.currentUser(), domain);
+ }
+ launcher.setDockerHostname(hostname);
+ launcher.setRunPrivilegedContainer(
+ compInstance.getCompSpec().getRunPrivilegedContainer());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
new file mode 100644
index 0000000..01f7b20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class TarballClientProvider extends AbstractClientProvider
+ implements YarnServiceConstants {
+
+ public TarballClientProvider() {
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fs)
+ throws IOException {
+ if (artifact == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(artifact.getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ Path p = new Path(artifact.getId());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException( "Artifact tarball does not exist "
+ + artifact.getId());
+ }
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ // validate dest_file is not absolute
+ if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Dest_file must not be absolute path: " + configFile.getDestFile());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
new file mode 100644
index 0000000..9d81f66
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public class TarballProviderFactory extends ProviderFactory {
+ private static final ProviderFactory FACTORY = new
+ TarballProviderFactory();
+
+ private TarballProviderFactory() {
+ }
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new TarballClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new TarballProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static ProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
new file mode 100644
index 0000000..9f29c8b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+
+public class TarballProviderService extends AbstractProviderService {
+
+ @Override
+ public void processArtifact(AbstractLauncher launcher,
+ ComponentInstance instance, SliderFileSystem fileSystem,
+ Service service)
+ throws IOException {
+ Path artifact = new Path(instance.getCompSpec().getArtifact().getId());
+ if (!fileSystem.isFile(artifact)) {
+ throw new IOException(
+ "Package doesn't exist as a resource: " + artifact.toString());
+ }
+ log.info("Adding resource {}", artifact.toString());
+ LocalResourceType type = LocalResourceType.ARCHIVE;
+ LocalResource packageResource = fileSystem.createAmResource(artifact, type);
+ launcher.addLocalResource(APP_LIB_DIR, packageResource);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
new file mode 100644
index 0000000..56634f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+/**
+ * These are constants unique to the Slider AM
+ */
+public class CustomRegistryConstants {
+
+ public static final String MANAGEMENT_REST_API =
+ "classpath:org.apache.slider.management";
+
+ public static final String REGISTRY_REST_API =
+ "classpath:org.apache.slider.registry";
+
+ public static final String PUBLISHER_REST_API =
+ "classpath:org.apache.slider.publisher";
+
+ public static final String PUBLISHER_CONFIGURATIONS_API =
+ "classpath:org.apache.slider.publisher.configurations";
+
+ public static final String PUBLISHER_EXPORTS_API =
+ "classpath:org.apache.slider.publisher.exports";
+
+ public static final String PUBLISHER_DOCUMENTS_API =
+ "classpath:org.apache.slider.publisher.documents";
+
+ public static final String AGENT_SECURE_REST_API =
+ "classpath:org.apache.slider.agents.secure";
+
+ public static final String AGENT_ONEWAY_REST_API =
+ "classpath:org.apache.slider.agents.oneway";
+
+ public static final String AM_IPC_PROTOCOL =
+ "classpath:org.apache.slider.appmaster.ipc";
+
+ public static final String AM_REST_BASE =
+ "classpath:org.apache.slider.client.rest";
+
+ public static final String WEB_UI = "http://";
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
new file mode 100644
index 0000000..fb2fd16
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -0,0 +1,654 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.service.api.ServiceApiConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.ComponentEventType;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
+import org.apache.hadoop.yarn.util.BoundedAppender;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.text.MessageFormat;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
+import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+
+/**
+ *
+ */
+public class ServiceScheduler extends CompositeService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceScheduler.class);
+ private Service app;
+
+ // component_name -> component
+ private final Map<String, Component> componentsByName =
+ new ConcurrentHashMap<>();
+
+ // id - > component
+ protected final Map<Long, Component> componentsById =
+ new ConcurrentHashMap<>();
+
+ private final Map<ContainerId, ComponentInstance> liveInstances =
+ new ConcurrentHashMap<>();
+
+ private ServiceMetrics serviceMetrics;
+
+ private ServiceTimelinePublisher serviceTimelinePublisher;
+
+ // Global diagnostics that will be reported to RM on eRxit.
+ // The unit the number of characters. This will be limited to 64 * 1024
+ // characters.
+ private BoundedAppender diagnostics = new BoundedAppender(64 * 1024);
+
+ // A cache for loading config files from remote such as hdfs
+ public LoadingCache<ConfigFile, Object> configFileCache = null;
+
+ public ScheduledExecutorService executorService;
+ public Map<String, String> globalTokens = new HashMap<>();
+
+ private AMRMClientAsync<AMRMClient.ContainerRequest> amRMClient;
+ private NMClientAsync nmClient;
+ private AsyncDispatcher dispatcher;
+ AsyncDispatcher compInstanceDispatcher;
+ private YarnRegistryViewForProviders yarnRegistryOperations;
+ private ServiceContext context;
+ private ContainerLaunchService containerLaunchService;
+
+ public ServiceScheduler(ServiceContext context) {
+ super(context.service.getName());
+ this.context = context;
+ }
+
+ public void buildInstance(ServiceContext context, Configuration configuration)
+ throws YarnException {
+ app = context.service;
+ executorService = Executors.newScheduledThreadPool(10);
+ RegistryOperations registryClient = RegistryOperationsFactory
+ .createInstance("ServiceScheduler", configuration);
+ addIfService(registryClient);
+ yarnRegistryOperations =
+ createYarnRegistryOperations(context, registryClient);
+
+ // register metrics
+ serviceMetrics = ServiceMetrics
+ .register(app.getName(), "Metrics for service");
+ serviceMetrics.tag("type", "Metrics type [component or service]", "service");
+ serviceMetrics.tag("appId", "Service id for service", app.getId());
+
+ amRMClient = createAMRMClient();
+ addIfService(amRMClient);
+
+ nmClient = createNMClient();
+ addIfService(nmClient);
+
+ dispatcher = new AsyncDispatcher("Component dispatcher");
+ dispatcher.register(ComponentEventType.class,
+ new ComponentEventHandler());
+ dispatcher.setDrainEventsOnStop();
+ addIfService(dispatcher);
+
+ compInstanceDispatcher =
+ new AsyncDispatcher("CompInstance dispatcher");
+ compInstanceDispatcher.register(ComponentInstanceEventType.class,
+ new ComponentInstanceEventHandler());
+ addIfService(compInstanceDispatcher);
+ containerLaunchService = new ContainerLaunchService(context.fs);
+ addService(containerLaunchService);
+
+ if (YarnConfiguration.timelineServiceV2Enabled(configuration)) {
+ TimelineV2Client timelineClient = TimelineV2Client
+ .createTimelineClient(context.attemptId.getApplicationId());
+ amRMClient.registerTimelineV2Client(timelineClient);
+ serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+ addService(serviceTimelinePublisher);
+ DefaultMetricsSystem.instance().register("ServiceMetricsSink",
+ "For processing metrics to ATS",
+ new ServiceMetricsSink(serviceTimelinePublisher));
+ LOG.info("Timeline v2 is enabled.");
+ }
+
+ initGlobalTokensForSubstitute(context);
+ //substitute quicklinks
+ ProviderUtils.substituteMapWithTokens(app.getQuicklinks(), globalTokens);
+ createConfigFileCache(context.fs.getFileSystem());
+
+ createAllComponents();
+ }
+
+ protected YarnRegistryViewForProviders createYarnRegistryOperations(
+ ServiceContext context, RegistryOperations registryClient) {
+ return new YarnRegistryViewForProviders(registryClient,
+ RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, app.getName(),
+ context.attemptId);
+ }
+
+ protected NMClientAsync createNMClient() {
+ return NMClientAsync.createNMClientAsync(new NMClientCallback());
+ }
+
+ protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
+ return AMRMClientAsync
+ .createAMRMClientAsync(1000, new AMRMClientCallback());
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ try {
+ buildInstance(context, conf);
+ } catch (YarnException e) {
+ throw new YarnRuntimeException(e);
+ }
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void serviceStop() throws Exception {
+ LOG.info("Stopping service scheduler");
+
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+
+ DefaultMetricsSystem.shutdown();
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ serviceTimelinePublisher
+ .serviceAttemptUnregistered(context, diagnostics.toString());
+ }
+ // Cleanup each component instance. no need to release containers as
+ // they will be automatically released by RM
+ for (ComponentInstance instance : liveInstances.values()) {
+ instance.cleanupRegistryAndCompHdfsDir();
+ }
+ String msg = diagnostics.toString()
+ + "Navigate to the failed component for more details.";
+ amRMClient
+ .unregisterApplicationMaster(FinalApplicationStatus.ENDED, msg, "");
+ LOG.info("Service " + app.getName()
+ + " unregistered with RM, with attemptId = " + context.attemptId
+ + ", diagnostics = " + diagnostics);
+ super.serviceStop();
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ super.serviceStart();
+ InetSocketAddress bindAddress = context.clientAMService.getBindAddress();
+ RegisterApplicationMasterResponse response = amRMClient
+ .registerApplicationMaster(bindAddress.getHostName(),
+ bindAddress.getPort(), "N/A");
+ if (response.getClientToAMTokenMasterKey() != null
+ && response.getClientToAMTokenMasterKey().remaining() != 0) {
+ context.secretManager
+ .setMasterKey(response.getClientToAMTokenMasterKey().array());
+ }
+ registerServiceInstance(context.attemptId, app);
+
+ //TODO handle containers recover
+ }
+
+ private void recover() {
+
+ }
+
+ private void initGlobalTokensForSubstitute(ServiceContext context) {
+ // ZK
+ globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM, getConfig()
+ .getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
+ String user = null;
+ try {
+ user = UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ LOG.error("Failed to get user.", e);
+ }
+ globalTokens
+ .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkClusterPath(user, app.getName()));
+
+ globalTokens.put(ServiceApiConstants.USER, user);
+ String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);
+ if (dnsDomain != null && !dnsDomain.isEmpty()) {
+ globalTokens.put(ServiceApiConstants.DOMAIN, dnsDomain);
+ }
+ // HDFS
+ String clusterFs = getConfig().getTrimmed(FS_DEFAULT_NAME_KEY);
+ if (clusterFs != null && !clusterFs.isEmpty()) {
+ globalTokens.put(ServiceApiConstants.CLUSTER_FS_URI, clusterFs);
+ globalTokens.put(ServiceApiConstants.CLUSTER_FS_HOST,
+ URI.create(clusterFs).getHost());
+ }
+ globalTokens.put(SERVICE_HDFS_DIR, context.serviceHdfsDir);
+ // service name
+ globalTokens.put(SERVICE_NAME_LC, app.getName().toLowerCase());
+ globalTokens.put(SERVICE_NAME, app.getName());
+ }
+
+ private void createConfigFileCache(final FileSystem fileSystem) {
+ this.configFileCache =
+ CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES)
+ .build(new CacheLoader<ConfigFile, Object>() {
+ @Override public Object load(ConfigFile key) throws Exception {
+ switch (key.getType()) {
+ case HADOOP_XML:
+ try (FSDataInputStream input = fileSystem
+ .open(new Path(key.getSrcFile()))) {
+ org.apache.hadoop.conf.Configuration confRead =
+ new org.apache.hadoop.conf.Configuration(false);
+ confRead.addResource(input);
+ Map<String, String> map = new HashMap<>(confRead.size());
+ for (Map.Entry<String, String> entry : confRead) {
+ map.put(entry.getKey(), entry.getValue());
+ }
+ return map;
+ }
+ case TEMPLATE:
+ try (FSDataInputStream fileInput = fileSystem
+ .open(new Path(key.getSrcFile()))) {
+ return IOUtils.toString(fileInput);
+ }
+ default:
+ return null;
+ }
+ }
+ });
+ context.configCache = configFileCache;
+ }
+
+ private void registerServiceInstance(ApplicationAttemptId attemptId,
+ Service service) throws IOException {
+ LOG.info("Registering " + attemptId + ", " + service.getName()
+ + " into registry");
+ ServiceRecord serviceRecord = new ServiceRecord();
+ serviceRecord.set(YarnRegistryAttributes.YARN_ID,
+ attemptId.getApplicationId().toString());
+ serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.APPLICATION);
+ serviceRecord.description = "Yarn Service Master";
+
+ serviceRecord.addExternalEndpoint(RegistryTypeUtils
+ .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
+ context.clientAMService.getBindAddress()));
+
+ // set any provided attributes
+ setUserProvidedServiceRecordAttributes(service.getConfiguration(),
+ serviceRecord);
+
+ executorService.submit(new Runnable() {
+ @Override public void run() {
+ try {
+ yarnRegistryOperations.registerSelf(serviceRecord, true);
+ LOG.info("Registered service under {}; absolute path {}",
+ yarnRegistryOperations.getSelfRegistrationPath(),
+ yarnRegistryOperations.getAbsoluteSelfRegistrationPath());
+ boolean isFirstAttempt = 1 == attemptId.getAttemptId();
+ // delete the children in case there are any and this is an AM startup.
+ // just to make sure everything underneath is purged
+ if (isFirstAttempt) {
+ yarnRegistryOperations.deleteChildren(
+ yarnRegistryOperations.getSelfRegistrationPath(), true);
+ }
+ } catch (IOException e) {
+ LOG.error(
+ "Failed to register app " + app.getName() + " in registry");
+ }
+ }
+ });
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ serviceTimelinePublisher.serviceAttemptRegistered(app, getConfig());
+ }
+ }
+
+ private void setUserProvidedServiceRecordAttributes(
+ org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) {
+ String prefix = "service.record.attribute";
+ for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
+ if (entry.getKey().startsWith(prefix)) {
+ String key = entry.getKey().substring(prefix.length() + 1);
+ record.set(key, entry.getValue().trim());
+ }
+ }
+ }
+
+ private void createAllComponents() {
+ long allocateId = 0;
+
+ // sort components by dependencies
+ Collection<org.apache.hadoop.yarn.service.api.records.Component> sortedComponents =
+ ServiceApiUtil.sortByDependencies(app.getComponents());
+
+ for (org.apache.hadoop.yarn.service.api.records.Component compSpec : sortedComponents) {
+ Component component = new Component(compSpec, allocateId, context);
+ componentsById.put(allocateId, component);
+ componentsByName.put(component.getName(), component);
+ allocateId++;
+
+ // Trigger the component without dependencies
+ if (component.areDependenciesReady()) {
+ ComponentEvent event = new ComponentEvent(compSpec.getName(), FLEX)
+ .setDesired(compSpec.getNumberOfContainers());
+ component.handle(event);
+ }
+ }
+ }
+
+ private final class ComponentEventHandler
+ implements EventHandler<ComponentEvent> {
+ @Override
+ public void handle(ComponentEvent event) {
+ Component component = componentsByName.get(event.getName());
+
+ if (component == null) {
+ LOG.error("No component exists for " + event.getName());
+ return;
+ }
+ try {
+ component.handle(event);
+ } catch (Throwable t) {
+ LOG.error(MessageFormat
+ .format("[COMPONENT {0}]: Error in handling event type {1}",
+ component.getName(), event.getType()), t);
+ }
+ }
+ }
+
+ private final class ComponentInstanceEventHandler
+ implements EventHandler<ComponentInstanceEvent> {
+ @Override
+ public void handle(ComponentInstanceEvent event) {
+ ComponentInstance instance =
+ liveInstances.get(event.getContainerId());
+ if (instance == null) {
+ LOG.error("No component instance exists for " + event.getContainerId());
+ return;
+ }
+ try {
+ instance.handle(event);
+ } catch (Throwable t) {
+ LOG.error(instance.getCompInstanceId() +
+ ": Error in handling event type " + event.getType(), t);
+ }
+ }
+ }
+
+ class AMRMClientCallback extends AMRMClientAsync.AbstractCallbackHandler {
+
+ @Override
+ public void onContainersAllocated(List<Container> containers) {
+ LOG.info(containers.size() + " containers allocated. ");
+ for (Container container : containers) {
+ Component comp = componentsById.get(container.getAllocationRequestId());
+ ComponentEvent event =
+ new ComponentEvent(comp.getName(), CONTAINER_ALLOCATED)
+ .setContainer(container);
+ dispatcher.getEventHandler().handle(event);
+ LOG.info("[COMPONENT {}]: {} outstanding container requests.",
+ comp.getName(),
+ amRMClient.getMatchingRequests(container.getAllocationRequestId()).size());
+ // remove the corresponding request
+ Collection<AMRMClient.ContainerRequest> collection = amRMClient
+ .getMatchingRequests(container.getAllocationRequestId());
+ if (collection.iterator().hasNext()) {
+ AMRMClient.ContainerRequest request = collection.iterator().next();
+ amRMClient.removeContainerRequest(request);
+ }
+
+ }
+ }
+
+ @Override
+ public void onContainersCompleted(List<ContainerStatus> statuses) {
+ for (ContainerStatus status : statuses) {
+ ContainerId containerId = status.getContainerId();
+ ComponentInstance instance = liveInstances.get(status.getContainerId());
+ if (instance == null) {
+ LOG.error(
+ "Container {} Completed. No component instance exists. exitStatus={}. diagnostics={} ",
+ containerId, status.getExitStatus(), status.getDiagnostics());
+ return;
+ }
+ ComponentEvent event =
+ new ComponentEvent(instance.getCompName(), CONTAINER_COMPLETED)
+ .setStatus(status).setInstance(instance);
+ dispatcher.getEventHandler().handle(event);
+ }
+ }
+
+ @Override
+ public void onContainersUpdated(List<UpdatedContainer> containers) {
+ }
+
+ @Override public void onShutdownRequest() {
+ //Was used for non-work-preserving restart in YARN, should be deprecated.
+ }
+
+ @Override public void onNodesUpdated(List<NodeReport> updatedNodes) {
+ StringBuilder str = new StringBuilder();
+ str.append("Nodes updated info: ").append(System.lineSeparator());
+ for (NodeReport report : updatedNodes) {
+ str.append(report.getNodeId()).append(", state = ")
+ .append(report.getNodeState()).append(", healthDiagnostics = ")
+ .append(report.getHealthReport()).append(System.lineSeparator());
+ }
+ LOG.warn(str.toString());
+ }
+
+ @Override public float getProgress() {
+ // get running containers over desired containers
+ long total = 0;
+ for (org.apache.hadoop.yarn.service.api.records.Component component : app
+ .getComponents()) {
+ total += component.getNumberOfContainers();
+ }
+ // Probably due to user flexed down to 0
+ if (total == 0) {
+ return 100;
+ }
+ return Math.max((float) liveInstances.size() / total * 100, 100);
+ }
+
+ @Override public void onError(Throwable e) {
+ LOG.error("Error in AMRMClient callback handler ", e);
+ }
+ }
+
+
+ private class NMClientCallback extends NMClientAsync.AbstractCallbackHandler {
+
+ @Override public void onContainerStarted(ContainerId containerId,
+ Map<String, ByteBuffer> allServiceResponse) {
+ ComponentInstance instance = liveInstances.get(containerId);
+ if (instance == null) {
+ LOG.error("No component instance exists for " + containerId);
+ return;
+ }
+ ComponentEvent event =
+ new ComponentEvent(instance.getCompName(), CONTAINER_STARTED)
+ .setInstance(instance);
+ dispatcher.getEventHandler().handle(event);
+ }
+
+ @Override public void onContainerStatusReceived(ContainerId containerId,
+ ContainerStatus containerStatus) {
+
+ }
+
+ @Override public void onContainerStopped(ContainerId containerId) {
+
+ }
+
+ @Override
+ public void onStartContainerError(ContainerId containerId, Throwable t) {
+ ComponentInstance instance = liveInstances.get(containerId);
+ if (instance == null) {
+ LOG.error("No component instance exists for " + containerId);
+ return;
+ }
+ amRMClient.releaseAssignedContainer(containerId);
+ // After container released, it'll get CONTAINER_COMPLETED event from RM
+ // automatically which will trigger stopping COMPONENT INSTANCE
+ }
+
+ @Override public void onContainerResourceIncreased(ContainerId containerId,
+ Resource resource) {
+
+ }
+
+ @Override public void onGetContainerStatusError(ContainerId containerId,
+ Throwable t) {
+
+ }
+
+ @Override
+ public void onIncreaseContainerResourceError(ContainerId containerId,
+ Throwable t) {
+
+ }
+
+ @Override
+ public void onStopContainerError(ContainerId containerId, Throwable t) {
+
+ }
+ }
+
+ public ServiceMetrics getServiceMetrics() {
+ return serviceMetrics;
+ }
+
+ public AMRMClientAsync<AMRMClient.ContainerRequest> getAmRMClient() {
+ return amRMClient;
+ }
+
+ public NMClientAsync getNmClient() {
+ return nmClient;
+ }
+
+ public void addLiveCompInstance(ContainerId containerId,
+ ComponentInstance instance) {
+ liveInstances.put(containerId, instance);
+ }
+
+ public void removeLiveCompInstance(ContainerId containerId) {
+ liveInstances.remove(containerId);
+ }
+
+ public AsyncDispatcher getCompInstanceDispatcher() {
+ return compInstanceDispatcher;
+ }
+
+ public YarnRegistryViewForProviders getYarnRegistryOperations() {
+ return yarnRegistryOperations;
+ }
+
+ public ServiceTimelinePublisher getServiceTimelinePublisher() {
+ return serviceTimelinePublisher;
+ }
+
+ public Map<ContainerId, ComponentInstance> getLiveInstances() {
+ return liveInstances;
+ }
+
+ public ContainerLaunchService getContainerLaunchService() {
+ return containerLaunchService;
+ }
+
+ public ServiceContext getContext() {
+ return context;
+ }
+
+ public Map<String, Component> getAllComponents() {
+ return componentsByName;
+ }
+
+ public Service getApp() {
+ return app;
+ }
+
+ public AsyncDispatcher getDispatcher() {
+ return dispatcher;
+ }
+
+ public BoundedAppender getDiagnostics() {
+ return diagnostics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
new file mode 100644
index 0000000..0bfb220
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+
+/**
+ * This class defines constants that can be used in input spec for
+ * variable substitutions
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public interface ServiceApiConstants {
+
+ // Constants for service
+ String SERVICE_NAME = $("SERVICE_NAME");
+
+ String SERVICE_NAME_LC = $("SERVICE_NAME.lc");
+
+ String USER = $("USER");
+
+ String DOMAIN = $("DOMAIN");
+
+ // Constants for component
+ String COMPONENT_NAME = $("COMPONENT_NAME");
+
+ String COMPONENT_NAME_LC = $("COMPONENT_NAME.lc");
+
+ String COMPONENT_INSTANCE_NAME = $("COMPONENT_INSTANCE_NAME");
+
+ // Constants for component instance
+ String COMPONENT_ID = $("COMPONENT_ID");
+
+ String CONTAINER_ID = $("CONTAINER_ID");
+
+ // Constants for default cluster ZK
+ String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
+
+ // URI for the default cluster fs
+ String CLUSTER_FS_URI = $("CLUSTER_FS_URI");
+
+ // the host component of the cluster fs UI
+ String CLUSTER_FS_HOST = $("CLUSTER_FS_HOST");
+
+ // Path in zookeeper for a specific service
+ String SERVICE_ZK_PATH = $("SERVICE_ZK_PATH");
+
+ // Constants for service specific hdfs dir
+ String SERVICE_HDFS_DIR = $("SERVICE_HDFS_DIR");
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
new file mode 100644
index 0000000..4328c6f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Artifact of an service component.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Artifact of an service component")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Artifact implements Serializable {
+ private static final long serialVersionUID = 3608929500111099035L;
+
+ private String id = null;
+
+ public enum TypeEnum {
+ DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = TypeEnum.DOCKER;
+ private String uri = null;
+
+ /**
+ * Artifact id. Examples are package location uri for tarball based services,
+ * image name for docker, etc.
+ **/
+ public Artifact id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "Artifact id. Examples are package location uri for tarball based services, image name for docker, etc.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * Artifact type, like docker, tarball, etc. (optional).
+ **/
+ public Artifact type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact type, like docker, tarball, etc. (optional).")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ /**
+ * Artifact location to support multiple artifact stores (optional).
+ **/
+ public Artifact uri(String uri) {
+ this.uri = uri;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact location to support multiple artifact stores (optional).")
+ @JsonProperty("uri")
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Artifact artifact = (Artifact) o;
+ return Objects.equals(this.id, artifact.id)
+ && Objects.equals(this.type, artifact.type)
+ && Objects.equals(this.uri, artifact.uri);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, type, uri);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Artifact {\n");
+
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" uri: ").append(toIndentedString(uri)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
new file mode 100644
index 0000000..7ac86d4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.Serializable;
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class BaseResource implements Serializable {
+ private static final long serialVersionUID = 1492603053176889431L;
+
+ private String uri;
+
+ /**
+ * Resource location for a service, e.g.
+ * /ws/v1/services/helloworld
+ *
+ **/
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("BaseResource [uri=");
+ builder.append(uri);
+ builder.append("]");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
new file mode 100644
index 0000000..b0a8e82
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * One or more components of the service. If the service is HBase say,
+ * then the component can be a simple role like master or regionserver. If the
+ * service is a complex business webapp then a component can be other
+ * services say Kafka or Storm. Thereby it opens up the support for complex
+ * and nested services.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Component implements Serializable {
+ private static final long serialVersionUID = -8430058381509087805L;
+
+ private String name = null;
+ private List<String> dependencies = new ArrayList<String>();
+ private ReadinessCheck readinessCheck = null;
+ private Artifact artifact = null;
+ private String launchCommand = null;
+ private Resource resource = null;
+ private Long numberOfContainers = null;
+ private Boolean runPrivilegedContainer = false;
+ private PlacementPolicy placementPolicy = null;
+ private Configuration configuration = new Configuration();
+ private List<String> quicklinks = new ArrayList<String>();
+ private List<Container> containers =
+ Collections.synchronizedList(new ArrayList<Container>());
+
+ /**
+ * Name of the service component (mandatory).
+ **/
+ public Component name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "Name of the service component (mandatory).")
+ @JsonProperty("name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * An array of service components which should be in READY state (as
+ * defined by readiness check), before this component can be started. The
+ * dependencies across all components of a service should be represented
+ * as a DAG.
+ **/
+ public Component dependencies(List<String> dependencies) {
+ this.dependencies = dependencies;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an service should be represented as a DAG.")
+ @JsonProperty("dependencies")
+ public List<String> getDependencies() {
+ return dependencies;
+ }
+
+ public void setDependencies(List<String> dependencies) {
+ this.dependencies = dependencies;
+ }
+
+ /**
+ * Readiness check for this component.
+ **/
+ public Component readinessCheck(ReadinessCheck readinessCheck) {
+ this.readinessCheck = readinessCheck;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Readiness check for this component.")
+ @JsonProperty("readiness_check")
+ public ReadinessCheck getReadinessCheck() {
+ return readinessCheck;
+ }
+
+ @XmlElement(name = "readiness_check")
+ public void setReadinessCheck(ReadinessCheck readinessCheck) {
+ this.readinessCheck = readinessCheck;
+ }
+
+ /**
+ * Artifact of the component (optional). If not specified, the service
+ * level global artifact takes effect.
+ **/
+ public Component artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the service level global artifact takes effect.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * The custom launch command of this component (optional). When specified at
+ * the component level, it overrides the value specified at the global level
+ * (if any).
+ **/
+ public Component launchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
+ @JsonProperty("launch_command")
+ public String getLaunchCommand() {
+ return launchCommand;
+ }
+
+ @XmlElement(name = "launch_command")
+ public void setLaunchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ }
+
+ /**
+ * Resource of this component (optional). If not specified, the service
+ * level global resource takes effect.
+ **/
+ public Component resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the service level global resource takes effect.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * Number of containers for this component (optional). If not specified,
+ * the service level global number_of_containers takes effect.
+ **/
+ public Component numberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.")
+ @JsonProperty("number_of_containers")
+ public Long getNumberOfContainers() {
+ return numberOfContainers;
+ }
+
+ @XmlElement(name = "number_of_containers")
+ public void setNumberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ }
+
+ @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
+ @JsonProperty("containers")
+ public List<Container> getContainers() {
+ return containers;
+ }
+
+ public void setContainers(List<Container> containers) {
+ this.containers = containers;
+ }
+
+ public void addContainer(Container container) {
+ this.containers.add(container);
+ }
+
+ public void removeContainer(Container container) {
+ containers.remove(container);
+ }
+ public Container getContainer(String id) {
+ for (Container container : containers) {
+ if (container.getId().equals(id)) {
+ return container;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Run all containers of this component in privileged mode (YARN-4262).
+ **/
+ public Component runPrivilegedContainer(Boolean runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
+ @JsonProperty("run_privileged_container")
+ public Boolean getRunPrivilegedContainer() {
+ return runPrivilegedContainer;
+ }
+
+ @XmlElement(name = "run_privileged_container")
+ public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ }
+
+ /**
+ * Advanced scheduling and placement policies for all containers of this
+ * component (optional). If not specified, the service level placement_policy
+ * takes effect. Refer to the description at the global level for more
+ * details.
+ **/
+ public Component placementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.")
+ @JsonProperty("placement_policy")
+ public PlacementPolicy getPlacementPolicy() {
+ return placementPolicy;
+ }
+
+ @XmlElement(name = "placement_policy")
+ public void setPlacementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ }
+
+ /**
+ * Config properties for this component.
+ **/
+ public Component configuration(Configuration configuration) {
+ this.configuration = configuration;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config properties for this component.")
+ @JsonProperty("configuration")
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * A list of quicklink keys defined at the service level, and to be
+ * resolved by this component.
+ **/
+ public Component quicklinks(List<String> quicklinks) {
+ this.quicklinks = quicklinks;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the service level, and to be resolved by this component.")
+ @JsonProperty("quicklinks")
+ public List<String> getQuicklinks() {
+ return quicklinks;
+ }
+
+ public void setQuicklinks(List<String> quicklinks) {
+ this.quicklinks = quicklinks;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Component component = (Component) o;
+ return Objects.equals(this.name, component.name)
+ && Objects.equals(this.dependencies, component.dependencies)
+ && Objects.equals(this.readinessCheck, component.readinessCheck)
+ && Objects.equals(this.artifact, component.artifact)
+ && Objects.equals(this.launchCommand, component.launchCommand)
+ && Objects.equals(this.resource, component.resource)
+ && Objects.equals(this.numberOfContainers, component.numberOfContainers)
+ && Objects.equals(this.runPrivilegedContainer,
+ component.runPrivilegedContainer)
+ && Objects.equals(this.placementPolicy, component.placementPolicy)
+ && Objects.equals(this.configuration, component.configuration)
+ && Objects.equals(this.quicklinks, component.quicklinks);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, dependencies, readinessCheck, artifact,
+ launchCommand, resource, numberOfContainers,
+ runPrivilegedContainer, placementPolicy, configuration, quicklinks);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Component {\n");
+
+ sb.append(" name: ").append(toIndentedString(name)).append("\n");
+ sb.append(" dependencies: ").append(toIndentedString(dependencies))
+ .append("\n");
+ sb.append(" readinessCheck: ").append(toIndentedString(readinessCheck))
+ .append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
+ .append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" numberOfContainers: ")
+ .append(toIndentedString(numberOfContainers)).append("\n");
+ sb.append(" containers: ").append(toIndentedString(containers))
+ .append("\n");
+ sb.append(" runPrivilegedContainer: ")
+ .append(toIndentedString(runPrivilegedContainer)).append("\n");
+ sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
+ .append("\n");
+ sb.append(" configuration: ").append(toIndentedString(configuration))
+ .append("\n");
+ sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
+ .append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ /**
+ * Merge from another component into this component without overwriting.
+ */
+ public void mergeFrom(Component that) {
+ if (this.getArtifact() == null) {
+ this.setArtifact(that.getArtifact());
+ }
+ if (this.getResource() == null) {
+ this.setResource(that.getResource());
+ }
+ if (this.getNumberOfContainers() == null) {
+ this.setNumberOfContainers(that.getNumberOfContainers());
+ }
+ if (this.getLaunchCommand() == null) {
+ this.setLaunchCommand(that.getLaunchCommand());
+ }
+ this.getConfiguration().mergeFrom(that.getConfiguration());
+ if (this.getQuicklinks() == null) {
+ this.setQuicklinks(that.getQuicklinks());
+ }
+ if (this.getRunPrivilegedContainer() == null) {
+ this.setRunPrivilegedContainer(that.getRunPrivilegedContainer());
+ }
+ if (this.getDependencies() == null) {
+ this.setDependencies(that.getDependencies());
+ }
+ if (this.getPlacementPolicy() == null) {
+ this.setPlacementPolicy(that.getPlacementPolicy());
+ }
+ if (this.getReadinessCheck() == null) {
+ this.setReadinessCheck(that.getReadinessCheck());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
new file mode 100644
index 0000000..c1502c7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A config file that needs to be created and made available as a volume in an
+ * service component container.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "A config file that needs to be created and made available as a volume in an service component container.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ConfigFile implements Serializable {
+ private static final long serialVersionUID = -7009402089417704612L;
+
+ public enum TypeEnum {
+ XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
+ "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = null;
+ private String destFile = null;
+ private String srcFile = null;
+ private Map<String, String> props = new HashMap<>();
+
+ public ConfigFile copy() {
+ ConfigFile copy = new ConfigFile();
+ copy.setType(this.getType());
+ copy.setSrcFile(this.getSrcFile());
+ copy.setDestFile(this.getDestFile());
+ if (this.getProps() != null && !this.getProps().isEmpty()) {
+ copy.getProps().putAll(this.getProps());
+ }
+ return copy;
+ }
+
+ /**
+ * Config file in the standard format like xml, properties, json, yaml,
+ * template.
+ **/
+ public ConfigFile type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config file in the standard format like xml, properties, json, yaml, template.")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ /**
+ * The absolute path that this configuration file should be mounted as, in the
+ * service container.
+ **/
+ public ConfigFile destFile(String destFile) {
+ this.destFile = destFile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The absolute path that this configuration file should be mounted as, in the service container.")
+ @JsonProperty("dest_file")
+ public String getDestFile() {
+ return destFile;
+ }
+
+ @XmlElement(name = "dest_file")
+ public void setDestFile(String destFile) {
+ this.destFile = destFile;
+ }
+
+ /**
+ * This provides the source location of the configuration file, the content
+ * of which is dumped to dest_file post property substitutions, in the format
+ * as specified in type. Typically the src_file would point to a source
+ * controlled network accessible file maintained by tools like puppet, chef,
+ * or hdfs etc. Currently, only hdfs is supported.
+ **/
+ public ConfigFile srcFile(String srcFile) {
+ this.srcFile = srcFile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "This provides the source location of the configuration file, "
+ + "the content of which is dumped to dest_file post property substitutions, in the format as specified in type. "
+ + "Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.")
+ @JsonProperty("src_file")
+ public String getSrcFile() {
+ return srcFile;
+ }
+
+ @XmlElement(name = "src_file")
+ public void setSrcFile(String srcFile) {
+ this.srcFile = srcFile;
+ }
+
+ /**
+ A blob of key value pairs that will be dumped in the dest_file in the format
+ as specified in type. If src_file is specified, src_file content are dumped
+ in the dest_file and these properties will overwrite, if any, existing
+ properties in src_file or be added as new properties in src_file.
+ **/
+ public ConfigFile props(Map<String, String> props) {
+ this.props = props;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type."
+ + " If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any,"
+ + " existing properties in src_file or be added as new properties in src_file.")
+ @JsonProperty("props")
+ public Map<String, String> getProps() {
+ return props;
+ }
+
+ public void setProps(Map<String, String> props) {
+ this.props = props;
+ }
+
+ public long getLong(String name, long defaultValue) {
+ if (name == null) {
+ return defaultValue;
+ }
+ String value = props.get(name.trim());
+ return Long.parseLong(value);
+ }
+
+ public boolean getBoolean(String name, boolean defaultValue) {
+ if (name == null) {
+ return defaultValue;
+ }
+ return Boolean.valueOf(props.get(name.trim()));
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ConfigFile configFile = (ConfigFile) o;
+ return Objects.equals(this.type, configFile.type)
+ && Objects.equals(this.destFile, configFile.destFile)
+ && Objects.equals(this.srcFile, configFile.srcFile);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, destFile, srcFile, props);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ConfigFile {\n");
+
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" destFile: ").append(toIndentedString(destFile)).append("\n");
+ sb.append(" srcFile: ").append(toIndentedString(srcFile)).append("\n");
+ sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
new file mode 100644
index 0000000..e10305a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.util.Locale;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum ConfigFormat {
+
+ JSON("json"),
+ PROPERTIES("properties"),
+ XML("xml"),
+ HADOOP_XML("hadoop_xml"),
+ ENV("env"),
+ TEMPLATE("template"),
+ YAML("yaml"),
+ ;
+ ConfigFormat(String suffix) {
+ this.suffix = suffix;
+ }
+
+ private final String suffix;
+
+ public String getSuffix() {
+ return suffix;
+ }
+
+
+ @Override
+ public String toString() {
+ return suffix;
+ }
+
+ /**
+ * Get a matching format or null
+ * @param type
+ * @return the format
+ */
+ public static ConfigFormat resolve(String type) {
+ for (ConfigFormat format: values()) {
+ if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
+ return format;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
new file mode 100644
index 0000000..2f8ca96
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Set of configuration properties that can be injected into the service
+ * components via envs, files and custom pluggable helper docker containers.
+ * Files of several standard formats like xml, properties, json, yaml and
+ * templates will be supported.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Configuration implements Serializable {
+ private static final long serialVersionUID = -4330788704981074466L;
+
+ private Map<String, String> properties = new HashMap<String, String>();
+ private Map<String, String> env = new HashMap<String, String>();
+ private List<ConfigFile> files = new ArrayList<ConfigFile>();
+
+ /**
+ * A blob of key-value pairs of common service properties.
+ **/
+ public Configuration properties(Map<String, String> properties) {
+ this.properties = properties;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common service properties.")
+ @JsonProperty("properties")
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * A blob of key-value pairs which will be appended to the default system
+ * properties and handed off to the service at start time. All placeholder
+ * references to properties will be substituted before injection.
+ **/
+ public Configuration env(Map<String, String> env) {
+ this.env = env;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.")
+ @JsonProperty("env")
+ public Map<String, String> getEnv() {
+ return env;
+ }
+
+ public void setEnv(Map<String, String> env) {
+ this.env = env;
+ }
+
+ /**
+ * Array of list of files that needs to be created and made available as
+ * volumes in the service component containers.
+ **/
+ public Configuration files(List<ConfigFile> files) {
+ this.files = files;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the service component containers.")
+ @JsonProperty("files")
+ public List<ConfigFile> getFiles() {
+ return files;
+ }
+
+ public void setFiles(List<ConfigFile> files) {
+ this.files = files;
+ }
+
+ public long getPropertyLong(String name, long defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Long.parseLong(value);
+ }
+
+ public int getPropertyInt(String name, int defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Integer.parseInt(value);
+ }
+
+ public boolean getPropertyBool(String name, boolean defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Boolean.parseBoolean(value);
+ }
+
+ public String getProperty(String name, String defaultValue) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return value;
+ }
+
+ public void setProperty(String name, String value) {
+ properties.put(name, value);
+ }
+
+ public String getProperty(String name) {
+ return properties.get(name.trim());
+ }
+
+ public String getEnv(String name) {
+ return env.get(name.trim());
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Configuration configuration = (Configuration) o;
+ return Objects.equals(this.properties, configuration.properties)
+ && Objects.equals(this.env, configuration.env)
+ && Objects.equals(this.files, configuration.files);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(properties, env, files);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Configuration {\n");
+
+ sb.append(" properties: ").append(toIndentedString(properties))
+ .append("\n");
+ sb.append(" env: ").append(toIndentedString(env)).append("\n");
+ sb.append(" files: ").append(toIndentedString(files)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ /**
+ * Merge all properties and envs from that configuration to this configration.
+ * For ConfigFiles, all properties and envs of that ConfigFile are merged into
+ * this ConfigFile.
+ */
+ public synchronized void mergeFrom(Configuration that) {
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
+ .getProperties());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
+
+ Map<String, ConfigFile> thatMap = new HashMap<>();
+ for (ConfigFile file : that.getFiles()) {
+ thatMap.put(file.getDestFile(), file.copy());
+ }
+ for (ConfigFile thisFile : files) {
+ if(thatMap.containsKey(thisFile.getDestFile())) {
+ ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
+ thatFile.getProps());
+ thatMap.remove(thisFile.getDestFile());
+ }
+ }
+ // add remaining new files from that Configration
+ for (ConfigFile thatFile : thatMap.values()) {
+ files.add(thatFile.copy());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
new file mode 100644
index 0000000..cf8cd79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Container.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Date;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * An instance of a running service container.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "An instance of a running service container")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Container extends BaseResource {
+ private static final long serialVersionUID = -8955788064529288L;
+
+ private String id = null;
+ private Date launchTime = null;
+ private String ip = null;
+ private String hostname = null;
+ private String bareHost = null;
+ private ContainerState state = null;
+ private String componentName = null;
+ private Resource resource = null;
+ private Artifact artifact = null;
+ private Boolean privilegedContainer = null;
+
+ /**
+ * Unique container id of a running service, e.g.
+ * container_e3751_1458061340047_0008_01_000002.
+ **/
+ public Container id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z.
+ * This will most likely be different from cluster launch time.
+ **/
+ public Container launchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
+ @JsonProperty("launch_time")
+ public Date getLaunchTime() {
+ return launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ @XmlElement(name = "launch_time")
+ public void setLaunchTime(Date launchTime) {
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+ }
+
+ /**
+ * IP address of a running container, e.g. 172.31.42.141. The IP address and
+ * hostname attribute values are dependent on the cluster/docker network setup
+ * as per YARN-4007.
+ **/
+ public Container ip(String ip) {
+ this.ip = ip;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
+ @JsonProperty("ip")
+ public String getIp() {
+ return ip;
+ }
+
+ public void setIp(String ip) {
+ this.ip = ip;
+ }
+
+ /**
+ * Fully qualified hostname of a running container, e.g.
+ * ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and
+ * hostname attribute values are dependent on the cluster/docker network setup
+ * as per YARN-4007.
+ **/
+ public Container hostname(String hostname) {
+ this.hostname = hostname;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
+ @JsonProperty("hostname")
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ /**
+ * The bare node or host in which the container is running, e.g.
+ * cn008.example.com.
+ **/
+ public Container bareHost(String bareHost) {
+ this.bareHost = bareHost;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The bare node or host in which the container is running, e.g. cn008.example.com.")
+ @JsonProperty("bare_host")
+ public String getBareHost() {
+ return bareHost;
+ }
+
+ @XmlElement(name = "bare_host")
+ public void setBareHost(String bareHost) {
+ this.bareHost = bareHost;
+ }
+
+ /**
+ * State of the container of an service.
+ **/
+ public Container state(ContainerState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "State of the container of an service.")
+ @JsonProperty("state")
+ public ContainerState getState() {
+ return state;
+ }
+
+ public void setState(ContainerState state) {
+ this.state = state;
+ }
+
+ /**
+ * Name of the component that this container instance belongs to.
+ **/
+ public Container componentName(String componentName) {
+ this.componentName = componentName;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Name of the component that this container instance belongs to.")
+ @JsonProperty("component_name")
+ public String getComponentName() {
+ return componentName;
+ }
+
+ @XmlElement(name = "component_name")
+ public void setComponentName(String componentName) {
+ this.componentName = componentName;
+ }
+
+ /**
+ * Resource used for this container.
+ **/
+ public Container resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource used for this container.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * Artifact used for this container.
+ **/
+ public Container artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact used for this container.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * Container running in privileged mode or not.
+ **/
+ public Container privilegedContainer(Boolean privilegedContainer) {
+ this.privilegedContainer = privilegedContainer;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Container running in privileged mode or not.")
+ @JsonProperty("privileged_container")
+ public Boolean getPrivilegedContainer() {
+ return privilegedContainer;
+ }
+
+ public void setPrivilegedContainer(Boolean privilegedContainer) {
+ this.privilegedContainer = privilegedContainer;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Container container = (Container) o;
+ return Objects.equals(this.id, container.id);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Container {\n");
+
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" launchTime: ").append(toIndentedString(launchTime))
+ .append("\n");
+ sb.append(" ip: ").append(toIndentedString(ip)).append("\n");
+ sb.append(" hostname: ").append(toIndentedString(hostname)).append("\n");
+ sb.append(" bareHost: ").append(toIndentedString(bareHost)).append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" componentName: ").append(toIndentedString(componentName))
+ .append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" privilegedContainer: ")
+ .append(toIndentedString(privilegedContainer)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
new file mode 100644
index 0000000..bf09ff2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current state of the container of an application.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum ContainerState {
+ RUNNING_BUT_UNREADY, READY, STOPPED
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/58] [abbrv] hadoop git commit: YARN-7393. RegistryDNS doesn't
work in tcp channel. Contributed by Eric Yang
Posted by ji...@apache.org.
YARN-7393. RegistryDNS doesn't work in tcp channel. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ec9e72c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ec9e72c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ec9e72c
Branch: refs/heads/yarn-native-services
Commit: 8ec9e72cc37957d8cf9af5a7b78086441ae64b00
Parents: d2775fb
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Oct 27 11:44:55 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:19 2017 -0800
----------------------------------------------------------------------
.../hadoop/registry/server/dns/RegistryDNS.java | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ec9e72c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 9097c46..d7ebece 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -189,8 +189,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
LOG.info("Opening TCP and UDP channels on {} port {}", addr, port);
addNIOUDP(addr, port);
- //TODO Fix dns lookup over TCP
-// addNIOTCP(addr, port);
+ addNIOTCP(addr, port);
}
/**
@@ -768,7 +767,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
buf.get(in, 0, messageLength);
Message query;
- byte[] response = null;
+ byte[] response;
try {
query = new Message(in);
LOG.info("received TCP query {}", query.getQuestion());
@@ -781,10 +780,19 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
ByteBuffer out = ByteBuffer.allocate(response.length + 2);
- out.putShort(0, (short) (response.length & 0xffff));
+ out.clear();
+ byte[] data = new byte[2];
+
+ data[1] = (byte)(response.length & 0xFF);
+ data[0] = (byte)((response.length >> 8) & 0xFF);
+ out.put(data);
out.put(response);
+ out.flip();
+
+ while(out.hasRemaining()) {
+ ch.write(out);
+ }
- ch.write(out);
} catch (IOException e) {
throw NetUtils.wrapException(ch.socket().getInetAddress().getHostName(),
ch.socket().getPort(),
@@ -836,6 +844,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
});
+ } else {
+ Thread.sleep(500);
}
}
} catch (IOException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[53/58] [abbrv] hadoop git commit: YARN-7326. Add recursion support
and configure RegistryDNS to lookup upstream. Contributed by Eric Yang
Posted by ji...@apache.org.
YARN-7326. Add recursion support and configure RegistryDNS to lookup upstream. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/501be9b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/501be9b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/501be9b4
Branch: refs/heads/yarn-native-services
Commit: 501be9b4be7d549b23cc501b26fdfdce0a7aa911
Parents: 68acd88
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Oct 25 07:37:03 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:19 2017 -0800
----------------------------------------------------------------------
.../hadoop/registry/server/dns/RegistryDNS.java | 148 ++++++++++++++++++-
.../registry/server/dns/TestRegistryDNS.java | 8 +-
2 files changed, 148 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/501be9b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index d7a415d..37e8429 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -37,8 +37,10 @@ import org.xbill.DNS.DNSKEYRecord;
import org.xbill.DNS.DNSSEC;
import org.xbill.DNS.DSRecord;
import org.xbill.DNS.ExtendedFlags;
+import org.xbill.DNS.ExtendedResolver;
import org.xbill.DNS.Flags;
import org.xbill.DNS.Header;
+import org.xbill.DNS.Lookup;
import org.xbill.DNS.Message;
import org.xbill.DNS.NSRecord;
import org.xbill.DNS.Name;
@@ -49,9 +51,12 @@ import org.xbill.DNS.RRSIGRecord;
import org.xbill.DNS.RRset;
import org.xbill.DNS.Rcode;
import org.xbill.DNS.Record;
+import org.xbill.DNS.Resolver;
+import org.xbill.DNS.ResolverConfig;
import org.xbill.DNS.SOARecord;
import org.xbill.DNS.Section;
import org.xbill.DNS.SetResponse;
+import org.xbill.DNS.SimpleResolver;
import org.xbill.DNS.TSIG;
import org.xbill.DNS.TSIGRecord;
import org.xbill.DNS.TextParseException;
@@ -66,8 +71,11 @@ import java.io.IOException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.NetworkInterface;
import java.net.Socket;
import java.net.SocketAddress;
+import java.net.SocketException;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.channels.DatagramChannel;
import java.nio.channels.ServerSocketChannel;
@@ -78,10 +86,13 @@ import java.security.PrivateKey;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.RSAPrivateKeySpec;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
+import java.util.Enumeration;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
@@ -134,6 +145,16 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
private boolean channelsInitialized = false;
/**
+ * Lock to update resolver only once per request.
+ */
+ private final Object resolverUpdateLock = new Object();
+
+ /**
+ * Whether resolver update has been requested.
+ */
+ private boolean resolverUpdateRequested = true;
+
+ /**
* Construct the service.
*
* @param name service name
@@ -172,6 +193,79 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
/**
+ * Initialize registryDNS to use /etc/resolv.conf values
+ * as default resolvers.
+ */
+ private void updateDNSServer(Configuration conf) {
+ synchronized (resolverUpdateLock) {
+ if (!resolverUpdateRequested) {
+ return;
+ }
+ int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT);
+ resolverUpdateRequested = false;
+ List<InetAddress> list = new ArrayList<InetAddress>();
+ try {
+ // If resolv.conf contains the server's own IP address,
+ // and RegistryDNS handles the lookup. Local IP address
+ // must be filter out from default resolvers to prevent
+ // self recursive loop.
+ if (port != 53) {
+ // When registryDNS is not running on default port,
+ // registryDNS can utilize local DNS server as upstream lookup.
+ throw new SocketException("Bypass filtering local DNS server.");
+ }
+ Enumeration<NetworkInterface> net =
+ NetworkInterface.getNetworkInterfaces();
+ while(net.hasMoreElements()) {
+ NetworkInterface n = (NetworkInterface) net.nextElement();
+ Enumeration<InetAddress> ee = n.getInetAddresses();
+ while (ee.hasMoreElements()) {
+ InetAddress i = (InetAddress) ee.nextElement();
+ list.add(i);
+ }
+ }
+ } catch (SocketException e) {
+ }
+ ResolverConfig.refresh();
+ ExtendedResolver resolver;
+ try {
+ resolver = new ExtendedResolver();
+ } catch (UnknownHostException e) {
+ LOG.error("Can not resolve DNS servers: ", e);
+ return;
+ }
+ for (Resolver check : resolver.getResolvers()) {
+ if (check instanceof SimpleResolver) {
+ InetAddress address = ((SimpleResolver) check).getAddress()
+ .getAddress();
+ if (list.contains(address)) {
+ resolver.deleteResolver(check);
+ continue;
+ } else {
+ check.setTimeout(30);
+ }
+ } else {
+ LOG.error("Not simple resolver!!!?" + check);
+ }
+ }
+ synchronized (Lookup.class) {
+ Lookup.setDefaultResolver(resolver);
+ Lookup.setDefaultSearchPath(ResolverConfig.getCurrentConfig()
+ .searchPath());
+ }
+ StringBuilder message = new StringBuilder();
+ message.append("DNS servers: ");
+ if (ResolverConfig.getCurrentConfig().servers() != null) {
+ for (String server : ResolverConfig.getCurrentConfig()
+ .servers()) {
+ message.append(server);
+ message.append(" ");
+ }
+ }
+ LOG.info(message.toString());
+ }
+ }
+ /**
* Initializes the registry.
*
* @param conf the hadoop configuration
@@ -183,6 +277,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
// create the zone. for now create a "dummy" SOA record
try {
+ updateDNSServer(conf);
setDomainName(conf);
initializeZones(conf);
@@ -916,9 +1011,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
for (int i = 0; i < 4; i++) {
response.removeAllRecords(i);
}
- if (rcode == Rcode.SERVFAIL) {
- response.addRecord(question, Section.QUESTION);
- }
+ response.addRecord(question, Section.QUESTION);
header.setRcode(rcode);
return response.toWire();
}
@@ -975,6 +1068,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
response.getHeader().setFlag(Flags.QR);
if (query.getHeader().getFlag(Flags.RD)) {
response.getHeader().setFlag(Flags.RD);
+ response.getHeader().setFlag(Flags.RA);
}
response.addRecord(queryRecord, Section.QUESTION);
@@ -992,10 +1086,10 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
LOG.debug("calling addAnswer");
byte rcode = addAnswer(response, name, type, dclass, 0, flags);
- if (rcode != Rcode.NOERROR && rcode != Rcode.NXDOMAIN) {
- return errorMessage(query, rcode);
+ if (rcode != Rcode.NOERROR) {
+ rcode = remoteLookup(response, name);
+ response.getHeader().setRcode(rcode);
}
-
addAdditional(response, flags);
if (queryOPT != null) {
@@ -1009,6 +1103,45 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
/**
+ * Lookup record from upstream DNS servers.
+ */
+ private byte remoteLookup(Message response, Name name) {
+ // Forward lookup to primary DNS servers
+ Record[] answers = getRecords(name, Type.ANY);
+ try {
+ for (Record r : answers) {
+ if (r.getType() == Type.SOA) {
+ response.addRecord(r, Section.AUTHORITY);
+ } else {
+ response.addRecord(r, Section.ANSWER);
+ }
+ }
+ } catch (NullPointerException e) {
+ return Rcode.NXDOMAIN;
+ } catch (Throwable e) {
+ return Rcode.SERVFAIL;
+ }
+ return Rcode.NOERROR;
+ }
+
+ /**
+ * Requests records for the given resource name.
+ *
+ * @param name - query string
+ * @param type - type of DNS record to lookup
+ * @return DNS records
+ */
+ protected Record[] getRecords(Name name, int type) {
+ try {
+ return new Lookup(name, type).run();
+ } catch (NullPointerException |
+ ExceptionInInitializerError e) {
+ LOG.error("Fail to lookup: " + name, e);
+ }
+ return null;
+ }
+
+ /**
* Create a query to forward to the primary DNS server (if configured).
* NOTE: Experimental
*
@@ -1180,7 +1313,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
rcode = Rcode.NOTAUTH;
}
}
- LOG.info("found record? {}", sr != null && sr.isSuccessful());
+ LOG.info("found local record? {}", sr != null && sr.isSuccessful());
if (sr != null) {
if (sr.isCNAME()) {
@@ -1241,6 +1374,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
}
}
+
return rcode;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/501be9b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
index ac8d939..7c78161 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -333,7 +333,7 @@ public class TestRegistryDNS extends Assert {
query.addRecord(optRecord, Section.ADDITIONAL);
byte[] responseBytes = getRegistryDNS().generateReply(query, null);
Message response = new Message(responseBytes);
- assertEquals("No answer should be returned", Rcode.NOTAUTH,
+ assertEquals("Missing record should be: ", Rcode.NXDOMAIN,
response.getRcode());
}
@@ -601,6 +601,12 @@ public class TestRegistryDNS extends Assert {
assertEquals(4, registryDNS.getZoneCount());
}
+ @Test
+ public void testExampleDotCom() throws Exception {
+ Name name = Name.fromString("example.com.");
+ Record[] records = getRegistryDNS().getRecords(name, Type.SOA);
+ assertNotNull("example.com exists:", records);
+ }
public RegistryDNS getRegistryDNS() {
return registryDNS;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/58] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
new file mode 100644
index 0000000..be36335
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
+import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for ServiceApiUtil helper methods.
+ */
+public class TestServiceApiUtil {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestServiceApiUtil.class);
+ private static final String EXCEPTION_PREFIX = "Should have thrown " +
+ "exception: ";
+ private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
+ "exception: ";
+
+ private static final String LEN_64_STR =
+ "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
+
+ private static final YarnConfiguration CONF_DEFAULT_DNS = new
+ YarnConfiguration();
+ private static final YarnConfiguration CONF_DNS_ENABLED = new
+ YarnConfiguration();
+
+ @BeforeClass
+ public static void init() {
+ CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
+ }
+
+ @Test(timeout = 90000)
+ public void testResourceValidation() throws Exception {
+ assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
+ .length());
+
+ SliderFileSystem sfs = initMock(null);
+
+ Service app = new Service();
+
+ // no name
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no name");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
+ }
+
+ // bad format name
+ String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
+ for (String badName : badNames) {
+ app.setName(badName);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with bad name " + badName);
+ } catch (IllegalArgumentException e) {
+
+ }
+ }
+
+ // launch command not specified
+ app.setName(LEN_64_STR);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DEFAULT_DNS);
+ Assert.fail(EXCEPTION_PREFIX + "service with no launch command");
+ } catch (IllegalArgumentException e) {
+ assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
+ e.getMessage());
+ }
+
+ // launch command not specified
+ app.setName(LEN_64_STR.substring(0, RegistryConstants
+ .MAX_FQDN_LABEL_LENGTH));
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no launch command");
+ } catch (IllegalArgumentException e) {
+ assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
+ e.getMessage());
+ }
+
+ // resource not specified
+ app.setLaunchCommand("sleep 3600");
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no resource");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
+ DEFAULT_COMPONENT_NAME), e.getMessage());
+ }
+
+ // memory not specified
+ Resource res = new Resource();
+ app.setResource(res);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no memory");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
+ DEFAULT_COMPONENT_NAME), e.getMessage());
+ }
+
+ // invalid no of cpus
+ res.setMemory("100mb");
+ res.setCpus(-2);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(
+ EXCEPTION_PREFIX + "service with invalid no of cpus");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
+ DEFAULT_COMPONENT_NAME), e.getMessage());
+ }
+
+ // number of containers not specified
+ res.setCpus(2);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no container count");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .contains(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // specifying profile along with cpus/memory raises exception
+ res.setProfile("hbase_finance_large");
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX
+ + "service with resource profile along with cpus/memory");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(RestApiErrorMessages
+ .ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
+ DEFAULT_COMPONENT_NAME),
+ e.getMessage());
+ }
+
+ // currently resource profile alone is not supported.
+ // TODO: remove the next test once resource profile alone is supported.
+ res.setCpus(null);
+ res.setMemory(null);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with resource profile only");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
+ e.getMessage());
+ }
+
+ // unset profile here and add cpus/memory back
+ res.setProfile(null);
+ res.setCpus(2);
+ res.setMemory("2gb");
+
+ // null number of containers
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "null number of containers");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // negative number of containers
+ app.setNumberOfContainers(-1L);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
+ }
+
+ // everything valid here
+ app.setNumberOfContainers(5L);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ LOG.error("service attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testArtifacts() throws IOException {
+ SliderFileSystem sfs = initMock(null);
+
+ Service app = new Service();
+ app.setName("name");
+ Resource res = new Resource();
+ app.setResource(res);
+ res.setMemory("512M");
+ app.setNumberOfContainers(3L);
+
+ // no artifact id fails with default type
+ Artifact artifact = new Artifact();
+ app.setArtifact(artifact);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no artifact id");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // no artifact id fails with SERVICE type
+ artifact.setType(Artifact.TypeEnum.SERVICE);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no artifact id");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // no artifact id fails with TARBALL type
+ artifact.setType(Artifact.TypeEnum.TARBALL);
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with no artifact id");
+ } catch (IllegalArgumentException e) {
+ assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // everything valid here
+ artifact.setType(Artifact.TypeEnum.DOCKER);
+ artifact.setId("docker.io/centos:centos7");
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ LOG.error("service attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ // defaults assigned
+ assertEquals(app.getComponents().get(0).getName(),
+ DEFAULT_COMPONENT_NAME);
+ assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
+ }
+
+ private static Resource createValidResource() {
+ Resource res = new Resource();
+ res.setMemory("512M");
+ return res;
+ }
+
+ private static Component createValidComponent(String compName) {
+ Component comp = new Component();
+ comp.setName(compName);
+ comp.setResource(createValidResource());
+ comp.setNumberOfContainers(1L);
+ return comp;
+ }
+
+ private static Service createValidApplication(String compName) {
+ Service app = new Service();
+ app.setLaunchCommand("sleep 3600");
+ app.setName("name");
+ app.setResource(createValidResource());
+ app.setNumberOfContainers(1L);
+ if (compName != null) {
+ app.addComponent(createValidComponent(compName));
+ }
+ return app;
+ }
+
+ private static SliderFileSystem initMock(Service ext) throws IOException {
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ JsonSerDeser<Service> jsonSerDeser = createNiceMock(JsonSerDeser
+ .class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ if (ext != null) {
+ expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
+ .anyTimes();
+ }
+ replay(sfs, mockFs, jsonSerDeser);
+ ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+ return sfs;
+ }
+
+ @Test
+ public void testExternalApplication() throws IOException {
+ Service ext = createValidApplication("comp1");
+ SliderFileSystem sfs = initMock(ext);
+
+ Service app = createValidApplication(null);
+
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.SERVICE);
+ artifact.setId("id");
+ app.setArtifact(artifact);
+
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ assertEquals(1, app.getComponents().size());
+ assertNotNull(app.getComponent("comp1"));
+ }
+
+ @Test
+ public void testDuplicateComponents() throws IOException {
+ SliderFileSystem sfs = initMock(null);
+
+ String compName = "comp1";
+ Service app = createValidApplication(compName);
+ app.addComponent(createValidComponent(compName));
+
+ // duplicate component name fails
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with component collision");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Component name collision: " + compName, e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExternalDuplicateComponent() throws IOException {
+ Service ext = createValidApplication("comp1");
+ SliderFileSystem sfs = initMock(ext);
+
+ Service app = createValidApplication("comp1");
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.SERVICE);
+ artifact.setId("id");
+ app.getComponent("comp1").setArtifact(artifact);
+
+ // duplicate component name okay in the case of SERVICE component
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExternalComponent() throws IOException {
+ Service ext = createValidApplication("comp1");
+ SliderFileSystem sfs = initMock(ext);
+
+ Service app = createValidApplication("comp2");
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.SERVICE);
+ artifact.setId("id");
+ app.setArtifact(artifact);
+
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ assertEquals(1, app.getComponents().size());
+ // artifact ID not inherited from global
+ assertNotNull(app.getComponent("comp2"));
+
+ // set SERVICE artifact id on component
+ app.getComponent("comp2").setArtifact(artifact);
+
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ assertEquals(1, app.getComponents().size());
+ // original component replaced by external component
+ assertNotNull(app.getComponent("comp1"));
+ }
+
+ public static void verifyDependencySorting(List<Component> components,
+ Component... expectedSorting) {
+ Collection<Component> actualSorting = ServiceApiUtil.sortByDependencies(
+ components);
+ assertEquals(expectedSorting.length, actualSorting.size());
+ int i = 0;
+ for (Component component : actualSorting) {
+ assertEquals(expectedSorting[i++], component);
+ }
+ }
+
+ @Test
+ public void testDependencySorting() throws IOException {
+ Component a = new Component().name("a");
+ Component b = new Component().name("b");
+ Component c = new Component().name("c");
+ Component d = new Component().name("d").dependencies(Arrays.asList("c"));
+ Component e = new Component().name("e").dependencies(Arrays.asList("b",
+ "d"));
+
+ verifyDependencySorting(Arrays.asList(a, b, c), a, b, c);
+ verifyDependencySorting(Arrays.asList(c, a, b), c, a, b);
+ verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e);
+ verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e);
+
+ c.setDependencies(Arrays.asList("e"));
+ try {
+ verifyDependencySorting(Arrays.asList(a, b, c, d, e));
+ Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle");
+ } catch (IllegalArgumentException ex) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d,
+ e)), ex.getMessage());
+ }
+
+ SliderFileSystem sfs = initMock(null);
+ Service service = createValidApplication(null);
+ service.setComponents(Arrays.asList(c, d, e));
+ try {
+ ServiceApiUtil.validateAndResolveService(service, sfs,
+ CONF_DEFAULT_DNS);
+ Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies");
+ } catch (IllegalArgumentException ex) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex
+ .getMessage());
+ }
+ }
+
+ @Test
+ public void testInvalidComponent() throws IOException {
+ SliderFileSystem sfs = initMock(null);
+ testComponent(sfs);
+ }
+
+ @Test
+ public void testValidateCompName() {
+ String[] invalidNames = {
+ "EXAMPLE", // UPPER case not allowed
+ "example_app" // underscore not allowed.
+ };
+ for (String name : invalidNames) {
+ try {
+ ServiceApiUtil.validateNameFormat(name, new Configuration());
+ Assert.fail();
+ } catch (IllegalArgumentException ex) {
+ ex.printStackTrace();
+ }
+ }
+ }
+
+ private static void testComponent(SliderFileSystem sfs)
+ throws IOException {
+ int maxLen = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
+ assertEquals(19, Long.toString(Long.MAX_VALUE).length());
+ maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
+
+ String compName = LEN_64_STR.substring(0, maxLen + 1);
+ Service app = createValidApplication(null);
+ app.addComponent(createValidComponent(compName));
+
+ // invalid component name fails if dns is enabled
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "service with invalid component name");
+ } catch (IllegalArgumentException e) {
+ assertEquals(String.format(RestApiErrorMessages
+ .ERROR_COMPONENT_NAME_INVALID, maxLen, compName), e.getMessage());
+ }
+
+ // does not fail if dns is disabled
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DEFAULT_DNS);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ compName = LEN_64_STR.substring(0, maxLen);
+ app = createValidApplication(null);
+ app.addComponent(createValidComponent(compName));
+
+ // does not fail
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
new file mode 100644
index 0000000..63aa9c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.curator.test.TestingCluster;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
+import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
+
+/**
+ * End to end tests to test deploying services with MiniYarnCluster and a in-JVM
+ * ZK testing cluster.
+ */
+public class TestYarnNativeServices extends ServiceTestUtils{
+
+ private static final Log LOG =
+ LogFactory.getLog(TestYarnNativeServices.class);
+
+ private MiniYARNCluster yarnCluster = null;
+ private MiniDFSCluster hdfsCluster = null;
+ private FileSystem fs = null;
+ protected Configuration conf = null;
+ private static final int NUM_NMS = 1;
+ private File basedir;
+
+ @Rule
+ public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @Before
+ public void setup() throws Exception {
+ setupInternal(NUM_NMS);
+ }
+
+ private void setupInternal(int numNodeManager)
+ throws Exception {
+ LOG.info("Starting up YARN cluster");
+// Logger rootLogger = LogManager.getRootLogger();
+// rootLogger.setLevel(Level.DEBUG);
+ conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+ // reduce the teardown waiting time
+ conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
+ conf.set("yarn.log.dir", "target");
+ // mark if we need to launch the v1 timeline server
+ // disable aux-service based timeline aggregators
+ conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
+ conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
+ // Enable ContainersMonitorImpl
+ conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
+ LinuxResourceCalculatorPlugin.class.getName());
+ conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
+ ProcfsBasedProcessTree.class.getName());
+ conf.setBoolean(
+ YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true);
+ conf.setBoolean(TIMELINE_SERVICE_ENABLED, false);
+ conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
+ conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000);
+ conf.setLong(AM_RESOURCE_MEM, 526);
+ conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5);
+ // Disable vmem check to disallow NM killing the container
+ conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
+ conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
+ // setup zk cluster
+ TestingCluster zkCluster;
+ zkCluster = new TestingCluster(1);
+ zkCluster.start();
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
+ conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
+ LOG.info("ZK cluster: " + zkCluster.getConnectString());
+
+ fs = FileSystem.get(conf);
+ basedir = new File("target", "apps");
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+
+ if (yarnCluster == null) {
+ yarnCluster =
+ new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1,
+ numNodeManager, 1, 1);
+ yarnCluster.init(conf);
+ yarnCluster.start();
+
+ waitForNMsToRegister();
+
+ URL url = Thread.currentThread().getContextClassLoader()
+ .getResource("yarn-site.xml");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find 'yarn-site.xml' dummy file in classpath");
+ }
+ Configuration yarnClusterConfig = yarnCluster.getConfig();
+ yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ new File(url.getPath()).getParent());
+ //write the document to a buffer (not directly to the file, as that
+ //can cause the file being written to get read -which will then fail.
+ ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+ yarnClusterConfig.writeXml(bytesOut);
+ bytesOut.close();
+ //write the bytes to the file in the classpath
+ OutputStream os = new FileOutputStream(new File(url.getPath()));
+ os.write(bytesOut.toByteArray());
+ os.close();
+ LOG.info("Write yarn-site.xml configs to: " + url);
+ }
+ if (hdfsCluster == null) {
+ HdfsConfiguration hdfsConfig = new HdfsConfiguration();
+ hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
+ .numDataNodes(1).build();
+ }
+
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
+ }
+
+
+ }
+
+ private void waitForNMsToRegister() throws Exception {
+ int sec = 60;
+ while (sec >= 0) {
+ if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size()
+ >= NUM_NMS) {
+ break;
+ }
+ Thread.sleep(1000);
+ sec--;
+ }
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (yarnCluster != null) {
+ try {
+ yarnCluster.stop();
+ } finally {
+ yarnCluster = null;
+ }
+ }
+ if (hdfsCluster != null) {
+ try {
+ hdfsCluster.shutdown();
+ } finally {
+ hdfsCluster = null;
+ }
+ }
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ SliderFileSystem sfs = new SliderFileSystem(conf);
+ Path appDir = sfs.getBaseApplicationPath();
+ sfs.getFileSystem().delete(appDir, true);
+ }
+
+
+
+ // End-to-end test to use ServiceClient to deploy a service.
+ // 1. Create a service with 2 components, each of which has 2 containers
+ // 2. Flex up each component to 3 containers and check the component instance names
+ // 3. Flex down each component to 1 container and check the component instance names
+ // 4. Flex up each component to 2 containers and check the component instance names
+ // 5. Stop the service
+ // 6. Destroy the service
+ @Test (timeout = 200000)
+ public void testCreateFlexStopDestroyService() throws Exception {
+ ServiceClient client = createClient();
+ Service exampleApp = createExampleApplication();
+ client.actionCreate(exampleApp);
+ SliderFileSystem fileSystem = new SliderFileSystem(conf);
+ Path appDir = fileSystem.buildClusterDirPath(exampleApp.getName());
+ // check app.json is persisted.
+ Assert.assertTrue(
+ fs.exists(new Path(appDir, exampleApp.getName() + ".json")));
+ waitForAllCompToBeReady(client, exampleApp);
+
+ // Flex two components, each from 2 container to 3 containers.
+ flexComponents(client, exampleApp, 3L);
+ // wait for flex to be completed, increase from 2 to 3 containers.
+ waitForAllCompToBeReady(client, exampleApp);
+ // check all instances name for each component are in sequential order.
+ checkCompInstancesInOrder(client, exampleApp);
+
+ // flex down to 1
+ flexComponents(client, exampleApp, 1L);
+ waitForAllCompToBeReady(client, exampleApp);
+ checkCompInstancesInOrder(client, exampleApp);
+
+ // check component dir and registry are cleaned up.
+
+ // flex up again to 2
+ flexComponents(client, exampleApp, 2L);
+ waitForAllCompToBeReady(client, exampleApp);
+ checkCompInstancesInOrder(client, exampleApp);
+
+ // stop the service
+ LOG.info("Stop the service");
+ client.actionStop(exampleApp.getName(), true);
+ ApplicationReport report = client.getYarnClient()
+ .getApplicationReport(ApplicationId.fromString(exampleApp.getId()));
+ // AM unregisters with RM successfully
+ Assert.assertEquals(FINISHED, report.getYarnApplicationState());
+ Assert.assertEquals(FinalApplicationStatus.ENDED,
+ report.getFinalApplicationStatus());
+
+ LOG.info("Destroy the service");
+ //destroy the service and check the app dir is deleted from fs.
+ client.actionDestroy(exampleApp.getName());
+ // check the service dir on hdfs (in this case, local fs) are deleted.
+ Assert.assertFalse(fs.exists(appDir));
+ }
+
+ // Create compa with 2 containers
+ // Create compb with 2 containers which depends on compa
+ // Check containers for compa started before containers for compb
+ @Test (timeout = 200000)
+ public void testComponentStartOrder() throws Exception {
+ ServiceClient client = createClient();
+ Service exampleApp = new Service();
+ exampleApp.setName("teststartorder");
+ exampleApp.addComponent(createComponent("compa", 2, "sleep 1000"));
+ Component compb = createComponent("compb", 2, "sleep 1000");
+
+ // Let compb depedends on compa;
+ compb.setDependencies(Collections.singletonList("compa"));
+ exampleApp.addComponent(compb);
+
+ client.actionCreate(exampleApp);
+ waitForAllCompToBeReady(client, exampleApp);
+
+ // check that containers for compa are launched before containers for compb
+ checkContainerLaunchDependencies(client, exampleApp, "compa", "compb");
+
+ client.actionStop(exampleApp.getName(), true);
+ client.actionDestroy(exampleApp.getName());
+ }
+
+ // Check containers launched are in dependency order
+ // Get all containers into a list and sort based on container launch time e.g.
+ // compa-c1, compa-c2, compb-c1, compb-c2;
+ // check that the container's launch time are align with the dependencies.
+ private void checkContainerLaunchDependencies(ServiceClient client,
+ Service exampleApp, String... compOrder)
+ throws IOException, YarnException {
+ Service retrievedApp = client.getStatus(exampleApp.getName());
+ List<Container> containerList = new ArrayList<>();
+ for (Component component : retrievedApp.getComponents()) {
+ containerList.addAll(component.getContainers());
+ }
+ // sort based on launchTime
+ containerList
+ .sort((o1, o2) -> o1.getLaunchTime().compareTo(o2.getLaunchTime()));
+ LOG.info("containerList: " + containerList);
+ // check the containers are in the dependency order.
+ int index = 0;
+ for (String comp : compOrder) {
+ long num = retrievedApp.getComponent(comp).getNumberOfContainers();
+ for (int i = 0; i < num; i++) {
+ String compInstanceName = containerList.get(index).getComponentName();
+ String compName =
+ compInstanceName.substring(0, compInstanceName.lastIndexOf('-'));
+ Assert.assertEquals(comp, compName);
+ index++;
+ }
+ }
+ }
+
+
+ private Map<String, Long> flexComponents(ServiceClient client,
+ Service exampleApp, long count) throws YarnException, IOException {
+ Map<String, Long> compCounts = new HashMap<>();
+ compCounts.put("compa", count);
+ compCounts.put("compb", count);
+ // flex will update the persisted conf to reflect latest number of containers.
+ exampleApp.getComponent("compa").setNumberOfContainers(count);
+ exampleApp.getComponent("compb").setNumberOfContainers(count);
+ client.flexByRestService(exampleApp.getName(), compCounts);
+ return compCounts;
+ }
+
+ // Check each component's comp instances name are in sequential order.
+ // E.g. If there are two instances compA-1 and compA-2
+ // When flex up to 4 instances, it should be compA-1 , compA-2, compA-3, compA-4
+ // When flex down to 3 instances, it should be compA-1 , compA-2, compA-3.
+ private void checkCompInstancesInOrder(ServiceClient client,
+ Service exampleApp) throws IOException, YarnException {
+ Service service = client.getStatus(exampleApp.getName());
+ for (Component comp : service.getComponents()) {
+ checkEachCompInstancesInOrder(comp);
+ }
+ }
+
+ private void checkRegistryAndCompDirDeleted() {
+
+ }
+
+ private void checkEachCompInstancesInOrder(Component component) {
+ long expectedNumInstances = component.getNumberOfContainers();
+ Assert.assertEquals(expectedNumInstances, component.getContainers().size());
+ TreeSet<String> instances = new TreeSet<>();
+ for (Container container : component.getContainers()) {
+ instances.add(container.getComponentName());
+ }
+
+ int i = 0;
+ for (String s : instances) {
+ Assert.assertEquals(component.getName() + "-" + i, s);
+ i++;
+ }
+ }
+
+ private void waitForOneCompToBeReady(ServiceClient client,
+ Service exampleApp, String readyComp)
+ throws TimeoutException, InterruptedException {
+ long numExpectedContainers =
+ exampleApp.getComponent(readyComp).getNumberOfContainers();
+ GenericTestUtils.waitFor(() -> {
+ try {
+ Service retrievedApp = client.getStatus(exampleApp.getName());
+ Component retrievedComp = retrievedApp.getComponent(readyComp);
+
+ if (retrievedComp.getContainers() != null
+ && retrievedComp.getContainers().size() == numExpectedContainers) {
+ LOG.info(readyComp + " found " + numExpectedContainers
+ + " containers running");
+ return true;
+ } else {
+ LOG.info(" Waiting for " + readyComp + "'s containers to be running");
+ return false;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }, 2000, 200000);
+ }
+
+ // wait until all the containers for all components become ready state
+ private void waitForAllCompToBeReady(ServiceClient client,
+ Service exampleApp) throws TimeoutException, InterruptedException {
+ int expectedTotalContainers = countTotalContainers(exampleApp);
+ GenericTestUtils.waitFor(() -> {
+ try {
+ Service retrievedApp = client.getStatus(exampleApp.getName());
+ int totalReadyContainers = 0;
+ LOG.info("Num Components " + retrievedApp.getComponents().size());
+ for (Component component : retrievedApp.getComponents()) {
+ LOG.info("looking for " + component.getName());
+ LOG.info(component);
+ if (component.getContainers() != null) {
+ if (component.getContainers().size() == exampleApp
+ .getComponent(component.getName()).getNumberOfContainers()) {
+ for (Container container : component.getContainers()) {
+ LOG.info(
+ "Container state " + container.getState() + ", component "
+ + component.getName());
+ if (container.getState() == ContainerState.READY) {
+ totalReadyContainers++;
+ LOG.info("Found 1 ready container " + container.getId());
+ }
+ }
+ } else {
+ LOG.info(component.getName() + " Expected number of containers "
+ + exampleApp.getComponent(component.getName())
+ .getNumberOfContainers() + ", current = " + component
+ .getContainers());
+ }
+ }
+ }
+ LOG.info("Exit loop, totalReadyContainers= " + totalReadyContainers
+ + " expected = " + expectedTotalContainers);
+ return totalReadyContainers == expectedTotalContainers;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }, 2000, 200000);
+ }
+
+ private ServiceClient createClient() throws Exception {
+ ServiceClient client = new ServiceClient() {
+ @Override protected Path addJarResource(String appName,
+ Map<String, LocalResource> localResources)
+ throws IOException, SliderException {
+ // do nothing, the Unit test will use local jars
+ return null;
+ }
+ };
+ client.init(conf);
+ client.start();
+ return client;
+ }
+
+
+ private int countTotalContainers(Service service) {
+ int totalContainers = 0;
+ for (Component component : service.getComponents()) {
+ totalContainers += component.getNumberOfContainers();
+ }
+ return totalContainers;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
new file mode 100644
index 0000000..93a15cc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
+import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
+
+/**
+ * Test for building / resolving components of type SERVICE.
+ */
+public class TestBuildExternalComponents {
+
+ protected Configuration conf = new YarnConfiguration();
+ private File basedir;
+
+ // Check component names match with expected
+ private static void checkComponentNames(List<Component> components,
+ Set<String> expectedComponents) {
+ Assert.assertEquals(expectedComponents.size(), components.size());
+ for (Component comp : components) {
+ Assert.assertTrue(expectedComponents.contains(comp.getName()));
+ }
+ }
+
+ // 1. Build the appDef and store on fs
+ // 2. check component names
+ private void buildAndCheckComponents(String appName, String appDef,
+ SliderFileSystem sfs, Set<String> names) throws Throwable {
+ String[] args =
+ { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
+ ClientArgs clientArgs = new ClientArgs(args);
+ clientArgs.parse();
+ ServiceCLI cli = new ServiceCLI() {
+ @Override protected void createServiceClient() {
+ client = new ServiceClient();
+ client.init(conf);
+ client.start();
+ }
+ };
+ cli.exec(clientArgs);
+
+ // verify generated conf
+ List<Component> components =
+ ServiceApiUtil.getComponents(sfs, appName);
+ checkComponentNames(components, names);
+ }
+
+ @Before
+ public void setup() throws IOException {
+ basedir = new File("target", "apps");
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ }
+
+ // Test applications defining external components(SERVICE type)
+ // can be resolved correctly
+ @Test
+ public void testExternalComponentBuild() throws Throwable {
+ SliderFileSystem sfs = new SliderFileSystem(conf);
+
+ Set<String> nameSet = new HashSet<>();
+ nameSet.add("simple");
+ nameSet.add("master");
+ nameSet.add("worker");
+
+ // app-1 has 3 components: simple, master, worker
+ buildAndCheckComponents("app-1", ExampleAppJson.APP_JSON, sfs, nameSet);
+ buildAndCheckComponents("external-0", ExampleAppJson.EXTERNAL_JSON_0, sfs,
+ nameSet);
+
+ nameSet.add("other");
+
+ // external1 has 3 components: simple(SERVICE - app1), master and other
+ buildAndCheckComponents("external-1", ExampleAppJson.EXTERNAL_JSON_1, sfs,
+ nameSet);
+
+ nameSet.add("another");
+
+ // external2 has 2 components: ext(SERVICE - external1), another
+ buildAndCheckComponents("external-2", ExampleAppJson.EXTERNAL_JSON_2, sfs,
+ nameSet);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
new file mode 100644
index 0000000..ecc529d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.util.Records;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
+import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.*;
+
+public class TestServiceCLI {
+
+ protected Configuration conf = new YarnConfiguration();
+ private File basedir;
+ private ServiceCLI cli;
+ private SliderFileSystem fs;
+
+ private void buildApp(String appName, String appDef) throws Throwable {
+ String[] args =
+ { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
+ ClientArgs clientArgs = new ClientArgs(args);
+ clientArgs.parse();
+ cli.exec(clientArgs);
+ }
+
+ @Before
+ public void setup() throws Throwable {
+ basedir = new File("target", "apps");
+ conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+ conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0);
+ conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1);
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
+ conf.setInt(CommonConfigurationKeysPublic.
+ IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
+ fs = new SliderFileSystem(conf);
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+
+ // create a CLI and skip connection to AM
+ cli = new ServiceCLI() {
+ @Override protected void createServiceClient() {
+ client = new ServiceClient() {
+ @Override
+ protected void serviceInit(Configuration configuration)
+ throws Exception {
+ super.serviceInit(conf);
+ yarnClient = spy(yarnClient);
+ ApplicationReport report = Records.newRecord(ApplicationReport.class);
+ report.setYarnApplicationState(YarnApplicationState.RUNNING);
+ report.setHost("localhost");
+ doReturn(report).when(yarnClient).getApplicationReport(anyObject());
+ }
+ @Override
+ protected ClientAMProtocol createAMProxy(String host, int port)
+ throws IOException {
+ return mock(ClientAMProtocol.class);
+ }
+ };
+ client.init(conf);
+ client.start();
+ }
+ };
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ }
+
+ // Test flex components count are persisted.
+ @Test
+ public void testFlexComponents() throws Throwable {
+ buildApp("service-1", ExampleAppJson.APP_JSON);
+
+ checkCompCount("master", 1L);
+
+ // increase by 2
+ String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"};
+ ClientArgs clientArgs = new ClientArgs(flexUpArgs);
+ clientArgs.parse();
+ cli.exec(clientArgs);
+ checkCompCount("master", 3L);
+
+ // decrease by 1
+ String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"};
+ clientArgs = new ClientArgs(flexDownArgs);
+ clientArgs.parse();
+ cli.exec(clientArgs);
+ checkCompCount("master", 2L);
+
+ String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"};
+ clientArgs = new ClientArgs(flexAbsoluteArgs);
+ clientArgs.parse();
+ cli.exec(clientArgs);
+ checkCompCount("master", 10L);
+ }
+
+ private void checkCompCount(String compName, long count) throws IOException {
+ List<Component> components =
+ ServiceApiUtil.getComponents(fs, "service-1");
+ for (Component component : components) {
+ if (component.getName().equals(compName)) {
+ Assert.assertEquals(count, component.getNumberOfContainers().longValue());
+ return;
+ }
+ }
+ Assert.fail();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
new file mode 100644
index 0000000..5fdd2ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
+
+/**
+ * Names of the example configs.
+ */
+public final class ExampleAppJson {
+
+ public static final String APP_JSON = "app.json";
+ public static final String OVERRIDE_JSON = "app-override.json";
+ public static final String DEFAULT_JSON = "default.json";
+ public static final String EXTERNAL_JSON_0 = "external0.json";
+ public static final String EXTERNAL_JSON_1 = "external1.json";
+ public static final String EXTERNAL_JSON_2 = "external2.json";
+
+ public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/";
+
+
+ private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON,
+ DEFAULT_JSON};
+
+ public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>();
+ static {
+ for (String example : ALL_EXAMPLES) {
+ ALL_EXAMPLE_RESOURCES.add(PACKAGE + example);
+ }
+ }
+
+ private ExampleAppJson() {
+ }
+
+ public static Service loadResource(String name) throws IOException {
+ return JSON_SER_DESER.fromResource(PACKAGE + name);
+ }
+
+ public static String resourceName(String name) {
+ return "target/test-classes" + PACKAGE + name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
new file mode 100644
index 0000000..04ec526
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*;
+import static org.easymock.EasyMock.*;
+
+/**
+ * Test global configuration resolution.
+ */
+public class TestAppJsonResolve extends Assert {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestAppJsonResolve.class);
+
+ @Test
+ public void testOverride() throws Throwable {
+ Service orig = ExampleAppJson.loadResource(OVERRIDE_JSON);
+
+ Configuration global = orig.getConfiguration();
+ assertEquals("a", global.getProperty("g1"));
+ assertEquals("b", global.getProperty("g2"));
+ assertEquals(2, global.getFiles().size());
+
+ Configuration simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(0, simple.getProperties().size());
+ assertEquals(1, simple.getFiles().size());
+
+ Configuration master = orig.getComponent("master").getConfiguration();
+ assertEquals("m", master.getProperty("name"));
+ assertEquals("overridden", master.getProperty("g1"));
+ assertEquals(0, master.getFiles().size());
+
+ Configuration worker = orig.getComponent("worker").getConfiguration();
+ LOG.info("worker = {}", worker);
+ assertEquals(3, worker.getProperties().size());
+ assertEquals(0, worker.getFiles().size());
+
+ assertEquals("worker", worker.getProperty("name"));
+ assertEquals("overridden-by-worker", worker.getProperty("g1"));
+ assertNull(worker.getProperty("g2"));
+ assertEquals("1000", worker.getProperty("timeout"));
+
+ // here is the resolution
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+ ServiceApiUtil.validateAndResolveService(orig, sfs, new
+ YarnConfiguration());
+
+ global = orig.getConfiguration();
+ LOG.info("global = {}", global);
+ assertEquals("a", global.getProperty("g1"));
+ assertEquals("b", global.getProperty("g2"));
+ assertEquals(2, global.getFiles().size());
+
+ simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(2, simple.getProperties().size());
+ assertEquals("a", simple.getProperty("g1"));
+ assertEquals("b", simple.getProperty("g2"));
+ assertEquals(2, simple.getFiles().size());
+
+ Set<ConfigFile> files = new HashSet<>();
+ Map<String, String> props = new HashMap<>();
+ props.put("k1", "overridden");
+ props.put("k2", "v2");
+ files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
+ .PROPERTIES).props(props));
+ files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
+ .XML).props(Collections.singletonMap("k3", "v3")));
+ assertTrue(files.contains(simple.getFiles().get(0)));
+ assertTrue(files.contains(simple.getFiles().get(1)));
+
+ master = orig.getComponent("master").getConfiguration();
+ LOG.info("master = {}", master);
+ assertEquals(3, master.getProperties().size());
+ assertEquals("m", master.getProperty("name"));
+ assertEquals("overridden", master.getProperty("g1"));
+ assertEquals("b", master.getProperty("g2"));
+ assertEquals(2, master.getFiles().size());
+
+ props.put("k1", "v1");
+ files.clear();
+ files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
+ .PROPERTIES).props(props));
+ files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
+ .XML).props(Collections.singletonMap("k3", "v3")));
+
+ assertTrue(files.contains(master.getFiles().get(0)));
+ assertTrue(files.contains(master.getFiles().get(1)));
+
+ worker = orig.getComponent("worker").getConfiguration();
+ LOG.info("worker = {}", worker);
+ assertEquals(4, worker.getProperties().size());
+
+ assertEquals("worker", worker.getProperty("name"));
+ assertEquals("overridden-by-worker", worker.getProperty("g1"));
+ assertEquals("b", worker.getProperty("g2"));
+ assertEquals("1000", worker.getProperty("timeout"));
+ assertEquals(2, worker.getFiles().size());
+
+ assertTrue(files.contains(worker.getFiles().get(0)));
+ assertTrue(files.contains(worker.getFiles().get(1)));
+ }
+
+ @Test
+ public void testOverrideExternalConfiguration() throws IOException {
+ Service orig = ExampleAppJson.loadResource(EXTERNAL_JSON_1);
+
+ Configuration global = orig.getConfiguration();
+ assertEquals(0, global.getProperties().size());
+
+ assertEquals(3, orig.getComponents().size());
+
+ Configuration simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(0, simple.getProperties().size());
+
+ Configuration master = orig.getComponent("master").getConfiguration();
+ assertEquals(1, master.getProperties().size());
+ assertEquals("is-overridden", master.getProperty("g3"));
+
+ Configuration other = orig.getComponent("other").getConfiguration();
+ assertEquals(0, other.getProperties().size());
+
+ // load the external service
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+ Service ext = ExampleAppJson.loadResource(APP_JSON);
+ ServiceApiUtil.validateAndResolveService(ext, sfs, new
+ YarnConfiguration());
+ reset(sfs, mockFs);
+
+ // perform the resolution on original service
+ JsonSerDeser<Service> jsonSerDeser = createNiceMock(JsonSerDeser
+ .class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
+ .anyTimes();
+ replay(sfs, mockFs, jsonSerDeser);
+ ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
+ ServiceApiUtil.validateAndResolveService(orig, sfs, new
+ YarnConfiguration());
+
+ global = orig.getConfiguration();
+ assertEquals(0, global.getProperties().size());
+
+ assertEquals(4, orig.getComponents().size());
+
+ simple = orig.getComponent("simple").getConfiguration();
+ assertEquals(3, simple.getProperties().size());
+ assertEquals("a", simple.getProperty("g1"));
+ assertEquals("b", simple.getProperty("g2"));
+ assertEquals("60",
+ simple.getProperty("yarn.service.failure-count-reset.window"));
+
+ master = orig.getComponent("master").getConfiguration();
+ assertEquals(5, master.getProperties().size());
+ assertEquals("512M", master.getProperty("jvm.heapsize"));
+ assertEquals("overridden", master.getProperty("g1"));
+ assertEquals("b", master.getProperty("g2"));
+ assertEquals("is-overridden", master.getProperty("g3"));
+ assertEquals("60",
+ simple.getProperty("yarn.service.failure-count-reset.window"));
+
+ Configuration worker = orig.getComponent("worker").getConfiguration();
+ LOG.info("worker = {}", worker);
+ assertEquals(4, worker.getProperties().size());
+ assertEquals("512M", worker.getProperty("jvm.heapsize"));
+ assertEquals("overridden-by-worker", worker.getProperty("g1"));
+ assertEquals("b", worker.getProperty("g2"));
+ assertEquals("60",
+ worker.getProperty("yarn.service.failure-count-reset.window"));
+
+ other = orig.getComponent("other").getConfiguration();
+ assertEquals(0, other.getProperties().size());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
new file mode 100644
index 0000000..83e9502
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
+import static org.easymock.EasyMock.*;
+
+/**
+ * Test loading example resources.
+ */
+@RunWith(value = Parameterized.class)
+public class TestLoadExampleAppJson extends Assert {
+ private String resource;
+
+ public TestLoadExampleAppJson(String resource) {
+ this.resource = resource;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<String[]> filenames() {
+ String[][] stringArray = new String[ExampleAppJson
+ .ALL_EXAMPLE_RESOURCES.size()][1];
+ int i = 0;
+ for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) {
+ stringArray[i++][0] = s;
+ }
+ return Arrays.asList(stringArray);
+ }
+
+ @Test
+ public void testLoadResource() throws Throwable {
+ try {
+ Service service = JSON_SER_DESER.fromResource(resource);
+
+ SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
+ expect(sfs.buildClusterDirPath(anyObject())).andReturn(
+ new Path("cluster_dir_path")).anyTimes();
+ replay(sfs, mockFs);
+
+ ServiceApiUtil.validateAndResolveService(service, sfs,
+ new YarnConfiguration());
+ } catch (Exception e) {
+ throw new Exception("exception loading " + resource + ":" + e.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
new file mode 100644
index 0000000..6159215
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test cluster name validation.
+ */
+public class TestValidateServiceNames {
+
+ void assertValidName(String name) {
+ ServiceApiUtil.validateNameFormat(name, new Configuration());
+ }
+
+ void assertInvalidName(String name) {
+ try {
+ ServiceApiUtil.validateNameFormat(name, new Configuration());
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ //
+ }
+ }
+
+ void assertInvalid(List<String> names) {
+ for (String name : names) {
+ assertInvalidName(name);
+ }
+ }
+
+ void assertValid(List<String> names) {
+ for (String name : names) {
+ assertValidName(name);
+ }
+ }
+
+ @Test
+ public void testEmptyName() throws Throwable {
+ assertInvalidName("");
+ }
+
+ @Test
+ public void testSpaceName() throws Throwable {
+ assertInvalidName(" ");
+ }
+
+
+ @Test
+ public void testLeadingHyphen() throws Throwable {
+ assertInvalidName("-hyphen");
+ }
+
+ @Test
+ public void testTitleLetters() throws Throwable {
+ assertInvalidName("Title");
+ }
+
+ @Test
+ public void testCapitalLetters() throws Throwable {
+ assertInvalidName("UPPER-CASE-CLUSTER");
+ }
+
+ @Test
+ public void testInnerBraced() throws Throwable {
+ assertInvalidName("a[a");
+ }
+
+ @Test
+ public void testLeadingBrace() throws Throwable {
+ assertInvalidName("[");
+ }
+
+ @Test
+ public void testNonalphaLeadingChars() throws Throwable {
+ assertInvalid(Arrays.asList(
+ "[a", "#", "@", "=", "*", "."
+ ));
+ }
+
+ @Test
+ public void testNonalphaInnerChars() throws Throwable {
+ assertInvalid(Arrays.asList(
+ "a[a", "b#", "c@", "d=", "e*", "f.", "g ", "h i"
+ ));
+ }
+
+ @Test
+ public void testClusterValid() throws Throwable {
+ assertValidName("cluster");
+ }
+
+ @Test
+ public void testValidNames() throws Throwable {
+ assertValid(Arrays.asList(
+ "cluster",
+ "cluster1",
+ "very-very-very-long-cluster-name",
+ "c1234567890"
+ ));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
new file mode 100644
index 0000000..0e03a2c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.yarn.service.monitor;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.MockServiceAM;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+
+public class TestServiceMonitor extends ServiceTestUtils {
+
+ private File basedir;
+ YarnConfiguration conf = new YarnConfiguration();
+
+ @Before
+ public void setup() throws Exception {
+ basedir = new File("target", "apps");
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+ conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 2);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ }
+
+ // Create compa with 1 container
+ // Create compb with 1 container
+ // Verify compb dependency satisfied
+ // Increase compa to 2 containers
+ // Verify compb dependency becomes unsatisfied.
+ @Test
+ public void testComponentDependency() throws Exception{
+ ApplicationId applicationId = ApplicationId.newInstance(123456, 1);
+ Service exampleApp = new Service();
+ exampleApp.setId(applicationId.toString());
+ exampleApp.setName("testComponentDependency");
+ exampleApp.addComponent(createComponent("compa", 1, "sleep 1000"));
+ Component compb = createComponent("compb", 1, "sleep 1000");
+
+ // Let compb depends on compa;
+ compb.setDependencies(Collections.singletonList("compa"));
+ exampleApp.addComponent(compb);
+
+ MockServiceAM am = new MockServiceAM(exampleApp);
+ am.init(conf);
+ am.start();
+
+ // compa ready
+ Assert.assertTrue(am.getComponent("compa").areDependenciesReady());
+ //compb not ready
+ Assert.assertFalse(am.getComponent("compb").areDependenciesReady());
+
+ // feed 1 container to compa,
+ am.feedContainerToComp(exampleApp, 1, "compa");
+ // waiting for compb's dependencies are satisfied
+ am.waitForDependenciesSatisfied("compb");
+
+ // feed 1 container to compb
+ am.feedContainerToComp(exampleApp, 2, "compb");
+ am.flexComponent("compa", 2);
+ am.waitForNumDesiredContainers("compa", 2);
+
+ // compb dependencies not satisfied again.
+ Assert.assertFalse(am.getComponent("compb").areDependenciesReady());
+ am.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
new file mode 100644
index 0000000..5b24a1d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestAbstractClientProvider.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.providers;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.easymock.EasyMock.*;
+
+/**
+ * Test the AbstractClientProvider shared methods.
+ */
+public class TestAbstractClientProvider {
+ private static final String EXCEPTION_PREFIX = "Should have thrown " +
+ "exception: ";
+ private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
+ "exception: ";
+
+ private static class ClientProvider extends AbstractClientProvider {
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem)
+ throws IOException {
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile,
+ FileSystem fileSystem) throws IOException {
+ }
+ }
+
+ @Test
+ public void testConfigFiles() throws IOException {
+ ClientProvider clientProvider = new ClientProvider();
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
+ replay(mockFs);
+
+ ConfigFile configFile = new ConfigFile();
+ List<ConfigFile> configFiles = new ArrayList<>();
+ configFiles.add(configFile);
+
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "null file type");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setType(ConfigFile.TypeEnum.TEMPLATE);
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "empty src_file for type template");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setSrcFile("srcfile");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "empty dest file");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setDestFile("destfile");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ configFile = new ConfigFile();
+ configFile.setType(ConfigFile.TypeEnum.JSON);
+ configFile.setSrcFile(null);
+ configFile.setDestFile("path/destfile2");
+ configFiles.add(configFile);
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "dest file with multiple path elements");
+ } catch (IllegalArgumentException e) {
+ }
+
+ configFile.setDestFile("/path/destfile2");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ configFile.setDestFile("destfile");
+ try {
+ clientProvider.validateConfigFiles(configFiles, mockFs);
+ Assert.fail(EXCEPTION_PREFIX + "duplicate dest file");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
new file mode 100644
index 0000000..56f4555
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/providers/TestProviderFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.providers;
+
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultClientProvider;
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderService;
+import org.apache.hadoop.yarn.service.provider.docker.DockerClientProvider;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderService;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballClientProvider;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderService;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test provider factories.
+ */
+public class TestProviderFactory {
+ @Test
+ public void testDockerFactory() throws Throwable {
+ ProviderFactory factory = ProviderFactory
+ .createServiceProviderFactory(new Artifact().type(TypeEnum.DOCKER));
+ assertTrue(factory instanceof DockerProviderFactory);
+ assertTrue(factory.createClientProvider() instanceof DockerClientProvider);
+ assertTrue(factory.createServerProvider() instanceof DockerProviderService);
+ assertTrue(ProviderFactory.getProviderService(new Artifact()
+ .type(TypeEnum.DOCKER)) instanceof DockerProviderService);
+ }
+
+ @Test
+ public void testTarballFactory() throws Throwable {
+ ProviderFactory factory = ProviderFactory
+ .createServiceProviderFactory(new Artifact().type(TypeEnum.TARBALL));
+ assertTrue(factory instanceof TarballProviderFactory);
+ assertTrue(factory.createClientProvider() instanceof TarballClientProvider);
+ assertTrue(factory.createServerProvider() instanceof
+ TarballProviderService);
+ assertTrue(ProviderFactory.getProviderService(new Artifact()
+ .type(TypeEnum.TARBALL)) instanceof TarballProviderService);
+ }
+
+ @Test
+ public void testDefaultFactory() throws Throwable {
+ ProviderFactory factory = ProviderFactory
+ .createServiceProviderFactory(null);
+ assertTrue(factory instanceof DefaultProviderFactory);
+ assertTrue(factory.createClientProvider() instanceof DefaultClientProvider);
+ assertTrue(factory.createServerProvider() instanceof DefaultProviderService);
+ assertTrue(ProviderFactory.getProviderService(null) instanceof
+ DefaultProviderService);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/58] [abbrv] hadoop git commit: YARN-7198. Add jsvc support for
RegistryDNS. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92734800
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92734800
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92734800
Branch: refs/heads/yarn-native-services
Commit: 92734800844b93bb19f3c0fca6be88d2801f7b1b
Parents: b57144a
Author: Jian He <ji...@apache.org>
Authored: Fri Oct 13 10:49:38 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:18 2017 -0800
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 6 ++
.../hadoop-yarn/conf/yarn-env.sh | 12 +++
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 73 +++++++++++++++++-
.../dns/PrivilegedRegistryDNSStarter.java | 80 ++++++++++++++++++++
.../hadoop/registry/server/dns/RegistryDNS.java | 45 +++++++----
.../registry/server/dns/RegistryDNSServer.java | 57 +++++---------
.../src/site/markdown/YarnCommands.md | 8 ++
.../src/site/markdown/registry/index.md | 1 +
.../markdown/registry/registry-configuration.md | 4 +-
.../site/markdown/yarn-service/RegistryDNS.md | 40 +++++++++-
.../markdown/yarn-service/ServiceDiscovery.md | 21 ++++-
11 files changed, 282 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 05d892b..727384c 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -45,6 +45,7 @@ function hadoop_usage
hadoop_add_subcommand "nodemanager" daemon "run a nodemanager on each worker"
hadoop_add_subcommand "proxyserver" daemon "run the web app proxy server"
hadoop_add_subcommand "queue" client "prints queue information"
+ hadoop_add_subcommand "registrydns" daemon "run the registry DNS server"
hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
@@ -143,6 +144,11 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
queue)
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI
;;
+ registrydns)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_SECURE_CLASSNAME='org.apache.hadoop.registry.server.dns.PrivilegedRegistryDNSStarter'
+ HADOOP_CLASSNAME='org.apache.hadoop.registry.server.dns.RegistryDNSServer'
+ ;;
resourcemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 90a87bf..4bd1d3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -160,3 +160,15 @@
# See ResourceManager for some examples
#
#export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')"
+
+###
+# Registry DNS specific parameters
+###
+# For privileged registry DNS, user to run as after dropping privileges
+# This will replace the hadoop.id.str Java property in secure mode.
+# export YARN_REGISTRYDNS_SECURE_USER=yarn
+
+# Supplemental options for privileged registry DNS
+# By default, Hadoop uses jsvc which needs to know to launch a
+# server jvm.
+# export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index e083312..4e805cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -34,6 +34,21 @@
<dependencies>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
</dependency>
@@ -70,14 +85,68 @@
</dependency>
<dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
- <scope>test</scope>
+ <artifactId>curator-recipes</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-daemon</groupId>
+ <artifactId>commons-daemon</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/PrivilegedRegistryDNSStarter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/PrivilegedRegistryDNSStarter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/PrivilegedRegistryDNSStarter.java
new file mode 100644
index 0000000..dd4e1b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/PrivilegedRegistryDNSStarter.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.commons.daemon.Daemon;
+import org.apache.commons.daemon.DaemonContext;
+import org.apache.hadoop.registry.client.api.DNSOperationsFactory;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.DEFAULT_DNS_PORT;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_PORT;
+
+/**
+ * This class is used to allow the RegistryDNSServer to run on a privileged
+ * port (e.g. 53).
+ */
+public class PrivilegedRegistryDNSStarter implements Daemon {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PrivilegedRegistryDNSStarter.class);
+
+ private YarnConfiguration conf;
+ private RegistryDNS registryDNS;
+ private RegistryDNSServer registryDNSServer;
+
+ @Override
+ public void init(DaemonContext context) throws Exception {
+ String[] args = context.getArguments();
+ StringUtils.startupShutdownMessage(RegistryDNSServer.class, args, LOG);
+ conf = new YarnConfiguration();
+ new GenericOptionsParser(conf, args);
+
+ int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT);
+ if (port < 1 || port > 1023) {
+ throw new RuntimeException("Must start privileged registry DNS server " +
+ "with '" + KEY_DNS_PORT + "' configured to a privileged port.");
+ }
+
+ try {
+ registryDNS = (RegistryDNS) DNSOperationsFactory.createInstance(conf);
+ registryDNS.initializeChannels(conf);
+ } catch (Exception e) {
+ LOG.error("Error initializing Registry DNS", e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void start() throws Exception {
+ registryDNSServer = RegistryDNSServer.launchDNSServer(conf, registryDNS);
+ }
+
+ @Override
+ public void stop() throws Exception {
+ }
+
+ @Override
+ public void destroy() {
+ registryDNSServer.stop();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 51139be..d7a415d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -82,6 +82,7 @@ import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
+import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
@@ -130,6 +131,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
private ConcurrentMap<Name, Zone> zones = new ConcurrentHashMap<>();
private Name bindHost;
+ private boolean channelsInitialized = false;
+
/**
* Construct the service.
*
@@ -150,6 +153,24 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
});
}
+ public void initializeChannels(Configuration conf) throws Exception {
+ if (channelsInitialized) {
+ return;
+ }
+ channelsInitialized = true;
+ int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT);
+ InetAddress addr = InetAddress.getLocalHost();
+
+ String bindAddress = conf.get(KEY_DNS_BIND_ADDRESS);
+ if (bindAddress != null) {
+ addr = InetAddress.getByName(bindAddress);
+ }
+
+ LOG.info("Opening TCP and UDP channels on {} port {}", addr, port);
+ addNIOUDP(addr, port);
+ addNIOTCP(addr, port);
+ }
+
/**
* Initializes the registry.
*
@@ -164,17 +185,9 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
try {
setDomainName(conf);
- int port = initializeZones(conf);
-
- InetAddress addr = InetAddress.getLocalHost();
-
- String bindAddress = conf.get(KEY_DNS_BIND_ADDRESS);
- if (bindAddress != null) {
- addr = InetAddress.getByName(bindAddress);
- }
- addNIOUDP(addr, port);
- addNIOTCP(addr, port);
+ initializeZones(conf);
+ initializeChannels(conf);
} catch (IOException e) {
LOG.error("Error initializing Registry DNS Server", e);
throw e;
@@ -189,8 +202,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
* @return the listener port
* @throws IOException
*/
- int initializeZones(Configuration conf) throws IOException {
- int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT);
+ void initializeZones(Configuration conf) throws IOException {
ttl = conf.getTimeDuration(KEY_DNS_TTL, 1L, TimeUnit.SECONDS);
RecordCreatorFactory.setTtl(ttl);
@@ -203,7 +215,12 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
initializeReverseLookupZone(conf);
- return port;
+ StringBuilder builder = new StringBuilder();
+ builder.append("DNS zones: ").append(System.lineSeparator());
+ for (Map.Entry<Name, Zone> entry : zones.entrySet()) {
+ builder.append(System.lineSeparator()).append(entry.getValue());
+ }
+ LOG.info(builder.toString());
}
/**
@@ -1412,7 +1429,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
processor.manageDNSRecords(command);
} else {
- LOG.warn("Yarn Resgistry record {} does not contain {} attribute ",
+ LOG.warn("Yarn Registry record {} does not contain {} attribute ",
record.toString(), YarnRegistryAttributes.YARN_PERSISTENCE);
}
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java
index faa5fe1..c7f6831 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java
@@ -17,11 +17,6 @@
package org.apache.hadoop.registry.server.dns;
import com.google.common.base.Preconditions;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.registry.client.api.DNSOperationsFactory;
@@ -65,9 +60,11 @@ public class RegistryDNSServer extends CompositeService {
/**
* Creates the DNS server.
* @param name the server name.
+ * @param registryDNS the registry DNS instance.
*/
- public RegistryDNSServer(String name) {
+ public RegistryDNSServer(String name, final RegistryDNS registryDNS) {
super(name);
+ this.registryDNS = registryDNS;
}
/**
@@ -83,8 +80,9 @@ public class RegistryDNSServer extends CompositeService {
registryOperations = new RegistryOperationsService("RegistryDNSOperations");
addService(registryOperations);
- // probably need to populate with existing apps?
- registryDNS = (RegistryDNS) DNSOperationsFactory.createInstance(conf);
+ if (registryDNS == null) {
+ registryDNS = (RegistryDNS) DNSOperationsFactory.createInstance(conf);
+ }
addService(registryDNS);
super.serviceInit(conf);
@@ -231,24 +229,21 @@ public class RegistryDNSServer extends CompositeService {
/**
* Launch the server.
- * @param args command line args.
+ * @param conf configuration
+ * @param rdns registry dns instance
* @return
*/
- static RegistryDNSServer launchDNSServer(String[] args) {
+ static RegistryDNSServer launchDNSServer(Configuration conf,
+ RegistryDNS rdns) {
RegistryDNSServer dnsServer = null;
Thread
.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
- StringUtils.startupShutdownMessage(RegistryDNSServer.class, args,
- LOG);
try {
- dnsServer = new RegistryDNSServer("RegistryDNSServer");
+ dnsServer = new RegistryDNSServer("RegistryDNSServer", rdns);
ShutdownHookManager.get().addShutdownHook(
new CompositeService.CompositeServiceShutdownHook(dnsServer),
SHUTDOWN_HOOK_PRIORITY);
- YarnConfiguration conf = new YarnConfiguration();
- processCommandLine(args, conf);
- new GenericOptionsParser(conf, args);
dnsServer.init(conf);
dnsServer.start();
} catch (Throwable t) {
@@ -259,32 +254,14 @@ public class RegistryDNSServer extends CompositeService {
}
/**
- * Process input command line arguments.
- * @param args the command line argument array.
- * @param conf the configuration.
- */
- private static void processCommandLine(String[] args,
- YarnConfiguration conf) {
- Options options = new Options();
- options.addOption("p", "port", true,
- "the server listening port (override)");
-
- CommandLineParser parser = new BasicParser();
- try {
- CommandLine cmd = parser.parse(options, args);
- if (cmd.hasOption("p")) {
- conf.set(RegistryConstants.KEY_DNS_PORT, cmd.getOptionValue("p"));
- }
- } catch (ParseException e) {
- LOG.error("Error parsing the command line options", e);
- }
- }
-
- /**
* Lanches the server instance.
* @param args the command line args.
+ * @throws IOException if command line options can't be parsed
*/
- public static void main(String[] args) {
- launchDNSServer(args);
+ public static void main(String[] args) throws IOException {
+ StringUtils.startupShutdownMessage(RegistryDNSServer.class, args, LOG);
+ YarnConfiguration conf = new YarnConfiguration();
+ new GenericOptionsParser(conf, args);
+ launchDNSServer(conf, null);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 51e27cc..6f00c5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -329,9 +329,17 @@ Usage: `yarn timelineserver`
Start the TimeLineServer
### apiserver
+
Usage: `yarn apiserver`
+
Start the API-server for deploying/managing services on YARN
+### registrydns
+
+Usage: `yarn registrydns`
+
+Start the RegistryDNS server
+
Files
-----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
index 3a648b6..0a127cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
@@ -22,6 +22,7 @@ to allow deployed applications to register themselves and the means of
communicating with them. Client applications can then locate services
and use the binding information to connect with the services's network-accessible
endpoints, be they REST, IPC, Web UI, Zookeeper quorum+path or some other protocol.
+Currently, all the registry data is stored in a zookeeper cluster.
* [Architecture](yarn-registry.html)
* [Configuration](registry-configuration.html)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md
index adca451..46bc92d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md
@@ -90,7 +90,7 @@ and the ports on which the ZK services are listening.
```
<property>
<description>
- List of hostname:port pairs defining the
+ A comma separated list of hostname:port pairs defining the
zookeeper quorum binding for the registry
</description>
<name>hadoop.registry.zk.quorum</name>
@@ -339,7 +339,7 @@ concluding that the quorum is unreachable and failing.
<property>
<description>
- List of hostname:port pairs defining the
+ A comma separated list of hostname:port pairs defining the
zookeeper quorum binding for the registry
</description>
<name>hadoop.registry.zk.quorum</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
index ef395fc..c7f19ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
@@ -15,7 +15,7 @@
# Registry DNS Server
<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-
+The document describes the internals of Registry DNS server. It is based on the [YARN service registry](../registry/index.md) which is backed by a zookeeper cluster.
## Introduction
The Registry DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
@@ -102,7 +102,7 @@ maps to the application user and so on. Wherever it is not easily distinguishabl
“container” or suffix such as “api”. For example, an endpoint published as a
management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*.
* Unique application name (per user) is not currently supported/guaranteed by YARN, but
-it is supported by frameworks such as Apache Slider. The registry DNS service currently
+it is supported by the YARN service framework. The registry DNS service currently
leverages the last element of the ZK path entry for the application as an
application name. These application names have to be unique for a given user.
@@ -153,6 +153,7 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm
| Name | Description |
| ------------ | ------------- |
+|hadoop.registry.zk.quorum| A comma separated list of hostname:port pairs defining the zookeeper quorum for the [YARN registry](../registry/registry-configuration.md). |
| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
@@ -163,4 +164,37 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm
| hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. |
| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. |
| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. |
-| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
\ No newline at end of file
+| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.|
+### Sample configurations
+```
+ <property>
+ <description>The domain name for Hadoop cluster associated records.</description>
+ <name>hadoop.registry.dns.domain-name</name>
+ <value>ycluster</value>
+ </property>
+
+ <property>
+ <description>The port number for the DNS listener. The default port is 5353.
+ If the standard privileged port 53 is used, make sure start the DNS with jsvc support.</description>
+ <name>hadoop.registry.dns.bind-port</name>
+ <value>5353</value>
+ </property>
+
+ <property>
+ <description>The DNS functionality is enabled for the cluster. Default is false.</description>
+ <name>hadoop.registry.dns.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>Address associated with the network interface to which the DNS listener should bind.</description>
+ <name>hadoop.registry.dns.bind-address</name>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <description>A comma separated list of hostname:port pairs defining the zookeeper quorum for the YARN registry</description>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ </property>
+ ```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
index a5dd0d2..ed56fa3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
@@ -73,6 +73,7 @@ assigned `0` since it is the first and only instance for the `hbasemaster` compo
Below is the set of configurations in `yarn-site.xml` required for enabling Registry DNS. A full list of properties can be found in the Configuration
section of [Registry DNS](RegistryDNS.md).
+
```
<property>
<description>The domain name for Hadoop cluster associated records.</description>
@@ -84,7 +85,7 @@ section of [Registry DNS](RegistryDNS.md).
<description>The port number for the DNS listener. The default port is 5353.
If the standard privileged port 53 is used, make sure start the DNS with jsvc support.</description>
<name>hadoop.registry.dns.bind-port</name>
- <value>53</value>
+ <value>5353</value>
</property>
<property>
@@ -94,6 +95,20 @@ section of [Registry DNS](RegistryDNS.md).
</property>
<property>
+ <description>Address associated with the network interface to which the DNS listener should bind.</description>
+ <name>hadoop.registry.dns.bind-address</name>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <description> A comma separated list of hostname:port pairs defining the zookeeper quorum for the YARN registry</description>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ </property>
+```
+To configure Registry DNS to serve reverse lookup for `172.17.0.0/24`
+```
+ <property>
<description>The network mask associated with the zone IP range. If specified, it is utilized to ascertain the
IP range possible and come up with an appropriate reverse zone name.</description>
<name>hadoop.registry.dns.zone-mask</name>
@@ -104,11 +119,9 @@ section of [Registry DNS](RegistryDNS.md).
<description>An indicator of the IP range associated with the cluster containers. The setting is utilized for the
generation of the reverse zone name.</description>
<name>hadoop.registry.dns.zone-subnet</name>
- <value>172.17.0</value>
+ <value>172.17.0.0</value>
</property>
-
```
-
## Start the DNS Server
By default, the DNS server runs on non-privileged port `5353`. Start the server
with:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/58] [abbrv] hadoop git commit: Rebased onto trunk - fix conflicts
Posted by ji...@apache.org.
Rebased onto trunk - fix conflicts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/307d55b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/307d55b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/307d55b3
Branch: refs/heads/yarn-native-services
Commit: 307d55b3e1ec9261c73e63286887c0b90a0c961e
Parents: 541b64c
Author: Jian He <ji...@apache.org>
Authored: Mon Sep 11 17:34:30 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:15 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/service/ServiceScheduler.java | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307d55b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index fb2fd16..7b809b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -574,6 +574,11 @@ public class ServiceScheduler extends CompositeService {
}
+ @Override public void onContainerResourceUpdated(ContainerId containerId,
+ Resource resource) {
+
+ }
+
@Override public void onGetContainerStatusError(ContainerId containerId,
Throwable t) {
@@ -586,6 +591,12 @@ public class ServiceScheduler extends CompositeService {
}
@Override
+ public void onUpdateContainerResourceError(ContainerId containerId,
+ Throwable t) {
+
+ }
+
+ @Override
public void onStopContainerError(ContainerId containerId, Throwable t) {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org