You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2018/05/18 21:30:50 UTC
[1/6] hadoop git commit: YARN-7530. Refactored YARN service API
project location. Contributed by Chandni Singh
Repository: hadoop
Updated Branches:
refs/heads/trunk 89f591139 -> a23ff8d88
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/example-app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/example-app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/example-app.json
new file mode 100644
index 0000000..a2f41cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/example-app.json
@@ -0,0 +1,16 @@
+{
+ "name": "example-app",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/log4j.properties
new file mode 100644
index 0000000..81a3f6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile
new file mode 100644
index 0000000..1d514d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "bad",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile
new file mode 100644
index 0000000..823561d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "example-app1",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile
new file mode 100644
index 0000000..823561d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "example-app1",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json
new file mode 100644
index 0000000..8a3a561
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json
@@ -0,0 +1,16 @@
+{
+ "name": "example-app3",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile
new file mode 100644
index 0000000..823561d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "example-app1",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile
new file mode 100644
index 0000000..d8fd1d1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "example-app2",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
index 5f5e70b..51e19b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
@@ -34,5 +34,6 @@
<modules>
<module>hadoop-yarn-services-core</module>
+ <module>hadoop-yarn-services-api</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index b2b34ec..490e9ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -37,7 +37,6 @@
<module>hadoop-yarn-applications-distributedshell</module>
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
<module>hadoop-yarn-services</module>
- <module>hadoop-yarn-services-api</module>
</modules>
<profiles>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/6] hadoop git commit: YARN-7530. Refactored YARN service API
project location. Contributed by Chandni Singh
Posted by ey...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
deleted file mode 100644
index 733b9bc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
+++ /dev/null
@@ -1,623 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import static org.junit.Assert.*;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.Path;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
-import com.google.common.collect.Sets;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.ComponentState;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.api.records.ServiceState;
-import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
-import org.apache.hadoop.yarn.service.conf.RestApiConstants;
-import org.apache.hadoop.yarn.service.webapp.ApiServer;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-/**
- * Test case for ApiServer REST API.
- *
- */
-public class TestApiServer {
- private ApiServer apiServer;
- private HttpServletRequest request;
- private ServiceClientTest mockServerClient;
-
- @Before
- public void setup() throws Exception {
- request = Mockito.mock(HttpServletRequest.class);
- Mockito.when(request.getRemoteUser())
- .thenReturn(System.getProperty("user.name"));
- mockServerClient = new ServiceClientTest();
- Configuration conf = new Configuration();
- conf.set("yarn.api-service.service.client.class",
- ServiceClientTest.class.getName());
- apiServer = new ApiServer(conf);
- apiServer.setServiceClient(mockServerClient);
- }
-
- @After
- public void teardown() {
- mockServerClient.forceStop();
- }
-
- @Test
- public void testPathAnnotation() {
- assertNotNull(this.apiServer.getClass().getAnnotation(Path.class));
- assertTrue("The controller has the annotation Path",
- this.apiServer.getClass().isAnnotationPresent(Path.class));
- final Path path = this.apiServer.getClass()
- .getAnnotation(Path.class);
- assertEquals("The path has /v1 annotation", "/v1", path.value());
- }
-
- @Test
- public void testGetVersion() {
- final Response actual = apiServer.getVersion();
- assertEquals("Version number is", Response.ok().build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testBadCreateService() {
- Service service = new Service();
- // Test for invalid argument
- final Response actual = apiServer.createService(request, service);
- assertEquals("Create service is ",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testGoodCreateService() throws Exception {
- String json = "{\"auths\": "
- + "{\"https://index.docker.io/v1/\": "
- + "{\"auth\": \"foobarbaz\"},"
- + "\"registry.example.com\": "
- + "{\"auth\": \"bazbarfoo\"}}}";
- File dockerTmpDir = new File("target", "docker-tmp");
- FileUtils.deleteQuietly(dockerTmpDir);
- dockerTmpDir.mkdirs();
- String dockerConfig = dockerTmpDir + "/config.json";
- BufferedWriter bw = new BufferedWriter(new FileWriter(dockerConfig));
- bw.write(json);
- bw.close();
- Service service = ServiceClientTest.buildGoodService();
- final Response actual = apiServer.createService(request, service);
- assertEquals("Create service is ",
- Response.status(Status.ACCEPTED).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testInternalServerErrorDockerClientConfigMissingCreateService() {
- Service service = new Service();
- service.setName("jenkins");
- service.setVersion("v1");
- service.setDockerClientConfig("/does/not/exist/config.json");
- Artifact artifact = new Artifact();
- artifact.setType(TypeEnum.DOCKER);
- artifact.setId("jenkins:latest");
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory("2048");
- List<Component> components = new ArrayList<>();
- Component c = new Component();
- c.setName("jenkins");
- c.setNumberOfContainers(1L);
- c.setArtifact(artifact);
- c.setLaunchCommand("");
- c.setResource(resource);
- components.add(c);
- service.setComponents(components);
- final Response actual = apiServer.createService(request, service);
- assertEquals("Create service is ",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testBadGetService() {
- final String serviceName = "nonexistent-jenkins";
- final Response actual = apiServer.getService(request, serviceName);
- assertEquals("Get service is ",
- Response.status(Status.NOT_FOUND).build().getStatus(),
- actual.getStatus());
- ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
- assertEquals("Response code don't match",
- RestApiConstants.ERROR_CODE_APP_NAME_INVALID, serviceStatus.getCode());
- assertEquals("Response diagnostics don't match",
- "Service " + serviceName + " not found",
- serviceStatus.getDiagnostics());
- }
-
- @Test
- public void testBadGetService2() {
- final Response actual = apiServer.getService(request, null);
- assertEquals("Get service is ",
- Response.status(Status.NOT_FOUND).build().getStatus(),
- actual.getStatus());
- ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
- assertEquals("Response code don't match",
- RestApiConstants.ERROR_CODE_APP_NAME_INVALID, serviceStatus.getCode());
- assertEquals("Response diagnostics don't match",
- "Service name cannot be null.", serviceStatus.getDiagnostics());
- }
-
- @Test
- public void testGoodGetService() {
- final Response actual = apiServer.getService(request, "jenkins");
- assertEquals("Get service is ",
- Response.status(Status.OK).build().getStatus(), actual.getStatus());
- }
-
- @Test
- public void testBadDeleteService() {
- final Response actual = apiServer.deleteService(request, "no-jenkins");
- assertEquals("Delete service is ",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testBadDeleteService2() {
- final Response actual = apiServer.deleteService(request, null);
- assertEquals("Delete service is ",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testBadDeleteService3() {
- final Response actual = apiServer.deleteService(request,
- "jenkins-doesn't-exist");
- assertEquals("Delete service is ",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testBadDeleteService4() {
- final Response actual = apiServer.deleteService(request,
- "jenkins-error-cleaning-registry");
- assertEquals("Delete service is ",
- Response.status(Status.INTERNAL_SERVER_ERROR).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testGoodDeleteService() {
- final Response actual = apiServer.deleteService(request, "jenkins");
- assertEquals("Delete service is ",
- Response.status(Status.OK).build().getStatus(), actual.getStatus());
- }
-
- @Test
- public void testDeleteStoppedService() {
- final Response actual = apiServer.deleteService(request,
- "jenkins-already-stopped");
- assertEquals("Delete service is ",
- Response.status(Status.OK).build().getStatus(), actual.getStatus());
- }
-
- @Test
- public void testDecreaseContainerAndStop() {
- Service service = new Service();
- service.setState(ServiceState.STOPPED);
- service.setName("jenkins");
- Artifact artifact = new Artifact();
- artifact.setType(TypeEnum.DOCKER);
- artifact.setId("jenkins:latest");
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory("2048");
- List<Component> components = new ArrayList<Component>();
- Component c = new Component();
- c.setName("jenkins");
- c.setNumberOfContainers(0L);
- c.setArtifact(artifact);
- c.setLaunchCommand("");
- c.setResource(resource);
- components.add(c);
- service.setComponents(components);
- final Response actual = apiServer.updateService(request, "jenkins",
- service);
- assertEquals("update service is ",
- Response.status(Status.OK).build().getStatus(), actual.getStatus());
- }
-
- @Test
- public void testBadDecreaseContainerAndStop() {
- Service service = new Service();
- service.setState(ServiceState.STOPPED);
- service.setName("no-jenkins");
- Artifact artifact = new Artifact();
- artifact.setType(TypeEnum.DOCKER);
- artifact.setId("jenkins:latest");
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory("2048");
- List<Component> components = new ArrayList<Component>();
- Component c = new Component();
- c.setName("no-jenkins");
- c.setNumberOfContainers(-1L);
- c.setArtifact(artifact);
- c.setLaunchCommand("");
- c.setResource(resource);
- components.add(c);
- service.setComponents(components);
- System.out.println("before stop");
- final Response actual = apiServer.updateService(request, "no-jenkins",
- service);
- assertEquals("flex service is ",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testIncreaseContainersAndStart() {
- Service service = new Service();
- service.setState(ServiceState.STARTED);
- service.setName("jenkins");
- Artifact artifact = new Artifact();
- artifact.setType(TypeEnum.DOCKER);
- artifact.setId("jenkins:latest");
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory("2048");
- List<Component> components = new ArrayList<Component>();
- Component c = new Component();
- c.setName("jenkins");
- c.setNumberOfContainers(2L);
- c.setArtifact(artifact);
- c.setLaunchCommand("");
- c.setResource(resource);
- components.add(c);
- service.setComponents(components);
- final Response actual = apiServer.updateService(request, "jenkins",
- service);
- assertEquals("flex service is ",
- Response.status(Status.OK).build().getStatus(), actual.getStatus());
- }
-
- @Test
- public void testBadStartServices() {
- Service service = new Service();
- service.setState(ServiceState.STARTED);
- service.setName("no-jenkins");
- Artifact artifact = new Artifact();
- artifact.setType(TypeEnum.DOCKER);
- artifact.setId("jenkins:latest");
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory("2048");
- List<Component> components = new ArrayList<Component>();
- Component c = new Component();
- c.setName("jenkins");
- c.setNumberOfContainers(2L);
- c.setArtifact(artifact);
- c.setLaunchCommand("");
- c.setResource(resource);
- components.add(c);
- service.setComponents(components);
- final Response actual = apiServer.updateService(request, "no-jenkins",
- service);
- assertEquals("start service is ",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testGoodStartServices() {
- Service service = new Service();
- service.setState(ServiceState.STARTED);
- service.setName("jenkins");
- Artifact artifact = new Artifact();
- artifact.setType(TypeEnum.DOCKER);
- artifact.setId("jenkins:latest");
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory("2048");
- List<Component> components = new ArrayList<Component>();
- Component c = new Component();
- c.setName("jenkins");
- c.setNumberOfContainers(2L);
- c.setArtifact(artifact);
- c.setLaunchCommand("");
- c.setResource(resource);
- components.add(c);
- service.setComponents(components);
- final Response actual = apiServer.updateService(request, "jenkins",
- service);
- assertEquals("start service is ",
- Response.status(Status.OK).build().getStatus(), actual.getStatus());
- }
-
- @Test
- public void testBadStopServices() {
- Service service = new Service();
- service.setState(ServiceState.STOPPED);
- service.setName("no-jenkins");
- Artifact artifact = new Artifact();
- artifact.setType(TypeEnum.DOCKER);
- artifact.setId("jenkins:latest");
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory("2048");
- List<Component> components = new ArrayList<Component>();
- Component c = new Component();
- c.setName("no-jenkins");
- c.setNumberOfContainers(-1L);
- c.setArtifact(artifact);
- c.setLaunchCommand("");
- c.setResource(resource);
- components.add(c);
- service.setComponents(components);
- System.out.println("before stop");
- final Response actual = apiServer.updateService(request, "no-jenkins",
- service);
- assertEquals("stop service is ",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testGoodStopServices() {
- Service service = new Service();
- service.setState(ServiceState.STOPPED);
- service.setName("jenkins");
- System.out.println("before stop");
- final Response actual = apiServer.updateService(request, "jenkins",
- service);
- assertEquals("stop service is ",
- Response.status(Status.OK).build().getStatus(), actual.getStatus());
- }
-
- @Test
- public void testBadSecondStopServices() throws Exception {
- Service service = new Service();
- service.setState(ServiceState.STOPPED);
- service.setName("jenkins-second-stop");
- // simulates stop on an already stopped service
- System.out.println("before second stop");
- final Response actual = apiServer.updateService(request,
- "jenkins-second-stop", service);
- assertEquals("stop service should have thrown 400 Bad Request: ",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
- assertEquals("Stop service should have failed with service already stopped",
- "Service jenkins-second-stop is already stopped",
- serviceStatus.getDiagnostics());
- }
-
- @Test
- public void testUpdateService() {
- Service service = new Service();
- service.setState(ServiceState.STARTED);
- service.setName("no-jenkins");
- Artifact artifact = new Artifact();
- artifact.setType(TypeEnum.DOCKER);
- artifact.setId("jenkins:latest");
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory("2048");
- List<Component> components = new ArrayList<Component>();
- Component c = new Component();
- c.setName("no-jenkins");
- c.setNumberOfContainers(-1L);
- c.setArtifact(artifact);
- c.setLaunchCommand("");
- c.setResource(resource);
- components.add(c);
- service.setComponents(components);
- System.out.println("before stop");
- final Response actual = apiServer.updateService(request, "no-jenkins",
- service);
- assertEquals("update service is ",
- Response.status(Status.BAD_REQUEST)
- .build().getStatus(), actual.getStatus());
- }
-
- @Test
- public void testUpdateComponent() {
- Response actual = apiServer.updateComponent(request, "jenkins",
- "jenkins-master", null);
- ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
- assertEquals("Update component should have failed with 400 bad request",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- assertEquals("Update component should have failed with no data error",
- "No component data provided", serviceStatus.getDiagnostics());
-
- Component comp = new Component();
- actual = apiServer.updateComponent(request, "jenkins", "jenkins-master",
- comp);
- serviceStatus = (ServiceStatus) actual.getEntity();
- assertEquals("Update component should have failed with 400 bad request",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- assertEquals("Update component should have failed with no count error",
- "No container count provided", serviceStatus.getDiagnostics());
-
- comp.setNumberOfContainers(-1L);
- actual = apiServer.updateComponent(request, "jenkins", "jenkins-master",
- comp);
- serviceStatus = (ServiceStatus) actual.getEntity();
- assertEquals("Update component should have failed with 400 bad request",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- assertEquals("Update component should have failed with no count error",
- "Invalid number of containers specified -1", serviceStatus.getDiagnostics());
-
- comp.setName("jenkins-slave");
- comp.setNumberOfContainers(1L);
- actual = apiServer.updateComponent(request, "jenkins", "jenkins-master",
- comp);
- serviceStatus = (ServiceStatus) actual.getEntity();
- assertEquals("Update component should have failed with 400 bad request",
- Response.status(Status.BAD_REQUEST).build().getStatus(),
- actual.getStatus());
- assertEquals(
- "Update component should have failed with component name mismatch "
- + "error",
- "Component name in the request object (jenkins-slave) does not match "
- + "that in the URI path (jenkins-master)",
- serviceStatus.getDiagnostics());
- }
-
- @Test
- public void testInitiateUpgrade() {
- Service goodService = ServiceClientTest.buildLiveGoodService();
- goodService.setVersion("v2");
- goodService.setState(ServiceState.UPGRADING);
- final Response actual = apiServer.updateService(request,
- goodService.getName(), goodService);
- assertEquals("Initiate upgrade is ",
- Response.status(Status.ACCEPTED).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testUpgradeSingleInstance() {
- Service goodService = ServiceClientTest.buildLiveGoodService();
- Component comp = goodService.getComponents().iterator().next();
- Container container = comp.getContainers().iterator().next();
- container.setState(ContainerState.UPGRADING);
-
- // To be able to upgrade, the service needs to be in UPGRADING
- // and container state needs to be in NEEDS_UPGRADE.
- Service serviceStatus = mockServerClient.getGoodServiceStatus();
- serviceStatus.setState(ServiceState.UPGRADING);
- Container liveContainer = serviceStatus.getComponents().iterator().next()
- .getContainers().iterator().next();
- liveContainer.setState(ContainerState.NEEDS_UPGRADE);
- mockServerClient.setExpectedInstances(Sets.newHashSet(
- liveContainer.getComponentInstanceName()));
-
- final Response actual = apiServer.updateComponentInstance(request,
- goodService.getName(), comp.getName(),
- container.getComponentInstanceName(), container);
- assertEquals("Instance upgrade is ",
- Response.status(Status.ACCEPTED).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testUpgradeMultipleInstances() {
- Service goodService = ServiceClientTest.buildLiveGoodService();
- Component comp = goodService.getComponents().iterator().next();
- comp.getContainers().forEach(container ->
- container.setState(ContainerState.UPGRADING));
-
- // To be able to upgrade, the service needs to be in UPGRADING
- // and container state needs to be in NEEDS_UPGRADE.
- Service serviceStatus = mockServerClient.getGoodServiceStatus();
- serviceStatus.setState(ServiceState.UPGRADING);
- Set<String> expectedInstances = new HashSet<>();
- serviceStatus.getComponents().iterator().next().getContainers().forEach(
- container -> {
- container.setState(ContainerState.NEEDS_UPGRADE);
- expectedInstances.add(container.getComponentInstanceName());
- }
- );
- mockServerClient.setExpectedInstances(expectedInstances);
-
- final Response actual = apiServer.updateComponentInstances(request,
- goodService.getName(), comp.getContainers());
- assertEquals("Instance upgrade is ",
- Response.status(Status.ACCEPTED).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testUpgradeComponent() {
- Service goodService = ServiceClientTest.buildLiveGoodService();
- Component comp = goodService.getComponents().iterator().next();
- comp.setState(ComponentState.UPGRADING);
-
- // To be able to upgrade, the service needs to be in UPGRADING
- // and component state needs to be in NEEDS_UPGRADE.
- Service serviceStatus = mockServerClient.getGoodServiceStatus();
- serviceStatus.setState(ServiceState.UPGRADING);
- Component liveComp = serviceStatus.getComponent(comp.getName());
- liveComp.setState(ComponentState.NEEDS_UPGRADE);
- Set<String> expectedInstances = new HashSet<>();
- liveComp.getContainers().forEach(container -> {
- expectedInstances.add(container.getComponentInstanceName());
- container.setState(ContainerState.NEEDS_UPGRADE);
- });
- mockServerClient.setExpectedInstances(expectedInstances);
-
- final Response actual = apiServer.updateComponent(request,
- goodService.getName(), comp.getName(), comp);
- assertEquals("Component upgrade is ",
- Response.status(Status.ACCEPTED).build().getStatus(),
- actual.getStatus());
- }
-
- @Test
- public void testUpgradeMultipleComps() {
- Service goodService = ServiceClientTest.buildLiveGoodService();
- goodService.getComponents().forEach(comp ->
- comp.setState(ComponentState.UPGRADING));
-
- // To be able to upgrade, the live service needs to be in UPGRADING
- // and component states needs to be in NEEDS_UPGRADE.
- Service serviceStatus = mockServerClient.getGoodServiceStatus();
- serviceStatus.setState(ServiceState.UPGRADING);
- Set<String> expectedInstances = new HashSet<>();
- serviceStatus.getComponents().forEach(liveComp -> {
- liveComp.setState(ComponentState.NEEDS_UPGRADE);
- liveComp.getContainers().forEach(liveContainer -> {
- expectedInstances.add(liveContainer.getComponentInstanceName());
- liveContainer.setState(ContainerState.NEEDS_UPGRADE);
- });
- });
- mockServerClient.setExpectedInstances(expectedInstances);
-
- final Response actual = apiServer.updateComponents(request,
- goodService.getName(), goodService.getComponents());
- assertEquals("Component upgrade is ",
- Response.status(Status.ACCEPTED).build().getStatus(),
- actual.getStatus());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
deleted file mode 100644
index 6cf0880..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
-
-/**
- * Test case for CLI to API Service.
- *
- */
-public class TestApiServiceClient {
- private static ApiServiceClient asc;
- private static ApiServiceClient badAsc;
- private static Server server;
-
- /**
- * A mocked version of API Service for testing purpose.
- *
- */
- @SuppressWarnings("serial")
- public static class TestServlet extends HttpServlet {
-
- @Override
- protected void doGet(HttpServletRequest req, HttpServletResponse resp)
- throws ServletException, IOException {
- System.out.println("Get was called");
- if (req.getPathInfo() != null
- && req.getPathInfo().contains("nonexistent-app")) {
- resp.setStatus(HttpServletResponse.SC_NOT_FOUND);
- } else {
- resp.setStatus(HttpServletResponse.SC_OK);
- }
- }
-
- @Override
- protected void doPost(HttpServletRequest req, HttpServletResponse resp)
- throws ServletException, IOException {
- resp.setStatus(HttpServletResponse.SC_OK);
- }
-
- @Override
- protected void doPut(HttpServletRequest req, HttpServletResponse resp)
- throws ServletException, IOException {
- resp.setStatus(HttpServletResponse.SC_OK);
- }
-
- @Override
- protected void doDelete(HttpServletRequest req, HttpServletResponse resp)
- throws ServletException, IOException {
- resp.setStatus(HttpServletResponse.SC_OK);
- }
-
- }
-
- @BeforeClass
- public static void setup() throws Exception {
- server = new Server(8088);
- ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(10);
- ServletContextHandler context = new ServletContextHandler();
- context.setContextPath("/app");
- server.setHandler(context);
- context.addServlet(new ServletHolder(TestServlet.class), "/*");
- ((ServerConnector)server.getConnectors()[0]).setHost("localhost");
- server.start();
-
- Configuration conf = new Configuration();
- conf.set("yarn.resourcemanager.webapp.address",
- "localhost:8088");
- asc = new ApiServiceClient();
- asc.serviceInit(conf);
-
- Configuration conf2 = new Configuration();
- conf2.set("yarn.resourcemanager.webapp.address",
- "localhost:8089");
- badAsc = new ApiServiceClient();
- badAsc.serviceInit(conf2);
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- server.stop();
- }
-
- @Test
- public void testLaunch() {
- String fileName = "target/test-classes/example-app.json";
- String appName = "example-app";
- long lifetime = 3600L;
- String queue = "default";
- try {
- int result = asc.actionLaunch(fileName, appName, lifetime, queue);
- assertEquals(EXIT_SUCCESS, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testBadLaunch() {
- String fileName = "unknown_file";
- String appName = "unknown_app";
- long lifetime = 3600L;
- String queue = "default";
- try {
- int result = badAsc.actionLaunch(fileName, appName, lifetime, queue);
- assertEquals(EXIT_EXCEPTION_THROWN, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testStatus() {
- String appName = "nonexistent-app";
- try {
- String result = asc.getStatusString(appName);
- assertEquals("Status reponse don't match",
- " Service " + appName + " not found", result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testStop() {
- String appName = "example-app";
- try {
- int result = asc.actionStop(appName);
- assertEquals(EXIT_SUCCESS, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testBadStop() {
- String appName = "unknown_app";
- try {
- int result = badAsc.actionStop(appName);
- assertEquals(EXIT_EXCEPTION_THROWN, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testStart() {
- String appName = "example-app";
- try {
- int result = asc.actionStart(appName);
- assertEquals(EXIT_SUCCESS, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testBadStart() {
- String appName = "unknown_app";
- try {
- int result = badAsc.actionStart(appName);
- assertEquals(EXIT_EXCEPTION_THROWN, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testSave() {
- String fileName = "target/test-classes/example-app.json";
- String appName = "example-app";
- long lifetime = 3600L;
- String queue = "default";
- try {
- int result = asc.actionSave(fileName, appName, lifetime, queue);
- assertEquals(EXIT_SUCCESS, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testBadSave() {
- String fileName = "unknown_file";
- String appName = "unknown_app";
- long lifetime = 3600L;
- String queue = "default";
- try {
- int result = badAsc.actionSave(fileName, appName, lifetime, queue);
- assertEquals(EXIT_EXCEPTION_THROWN, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testFlex() {
- String appName = "example-app";
- HashMap<String, String> componentCounts = new HashMap<String, String>();
- try {
- int result = asc.actionFlex(appName, componentCounts);
- assertEquals(EXIT_SUCCESS, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testBadFlex() {
- String appName = "unknown_app";
- HashMap<String, String> componentCounts = new HashMap<String, String>();
- try {
- int result = badAsc.actionFlex(appName, componentCounts);
- assertEquals(EXIT_EXCEPTION_THROWN, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testDestroy() {
- String appName = "example-app";
- try {
- int result = asc.actionDestroy(appName);
- assertEquals(EXIT_SUCCESS, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testBadDestroy() {
- String appName = "unknown_app";
- try {
- int result = badAsc.actionDestroy(appName);
- assertEquals(EXIT_EXCEPTION_THROWN, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testInitiateServiceUpgrade() {
- String appName = "example-app";
- String upgradeFileName = "target/test-classes/example-app.json";
- try {
- int result = asc.initiateUpgrade(appName, upgradeFileName, false);
- assertEquals(EXIT_SUCCESS, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testInstancesUpgrade() {
- String appName = "example-app";
- try {
- int result = asc.actionUpgradeInstances(appName, Lists.newArrayList(
- "comp-1", "comp-2"));
- assertEquals(EXIT_SUCCESS, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
- @Test
- public void testComponentsUpgrade() {
- String appName = "example-app";
- try {
- int result = asc.actionUpgradeComponents(appName, Lists.newArrayList(
- "comp"));
- assertEquals(EXIT_SUCCESS, result);
- } catch (IOException | YarnException e) {
- fail();
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
deleted file mode 100644
index d39083d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Test class for system service manager.
- */
-public class TestSystemServiceManagerImpl {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestSystemServiceManagerImpl.class);
- private SystemServiceManagerImpl systemService;
- private Configuration conf;
- private String resourcePath = "system-services";
-
- private String[] users = new String[] {"user1", "user2"};
- private static Map<String, Set<String>> loadedServices = new HashMap<>();
- private static Map<String, Set<String>> submittedServices = new HashMap<>();
-
- @Before
- public void setup() {
- File file = new File(
- getClass().getClassLoader().getResource(resourcePath).getFile());
- conf = new Configuration();
- conf.set(YarnServiceConf.YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY,
- file.getAbsolutePath());
- systemService = new SystemServiceManagerImpl() {
- @Override ServiceClient getServiceClient() {
- return new TestServiceClient();
- }
- };
- systemService.init(conf); // do not call explicit start
-
- constructUserService(users[0], "example-app1");
- constructUserService(users[1], "example-app1", "example-app2");
- }
-
- @After
- public void teadDown() {
- systemService.stop();
- }
-
- @Test
- public void testSystemServiceSubmission() throws Exception {
- systemService.start();
-
- /* verify for ignored sevices count */
- Map<String, Integer> ignoredUserServices =
- systemService.getIgnoredUserServices();
- Assert.assertEquals(1, ignoredUserServices.size());
- Assert.assertTrue("User user1 doesn't exist.",
- ignoredUserServices.containsKey(users[0]));
- int count = ignoredUserServices.get(users[0]);
- Assert.assertEquals(1, count);
- Assert.assertEquals(1,
- systemService.getBadFileNameExtensionSkipCounter());
- Assert.assertEquals(1, systemService.getBadDirSkipCounter());
-
- Map<String, Set<Service>> userServices =
- systemService.getSyncUserServices();
- Assert.assertEquals(loadedServices.size(), userServices.size());
- verifyForScannedUserServices(userServices);
-
- verifyForLaunchedUserServices();
-
- // 2nd time launch service to handle if service exist scenario
- systemService.launchUserService(userServices);
- verifyForLaunchedUserServices();
- }
-
- private void verifyForScannedUserServices(
- Map<String, Set<Service>> userServices) {
- for (String user : users) {
- Set<Service> services = userServices.get(user);
- Set<String> serviceNames = loadedServices.get(user);
- Assert.assertEquals(serviceNames.size(), services.size());
- Iterator<Service> iterator = services.iterator();
- while (iterator.hasNext()) {
- Service next = iterator.next();
- Assert.assertTrue(
- "Service name doesn't exist in expected userService "
- + serviceNames, serviceNames.contains(next.getName()));
- }
- }
- }
-
- public void constructUserService(String user, String... serviceNames) {
- Set<String> service = loadedServices.get(user);
- if (service == null) {
- service = new HashSet<>();
- for (String serviceName : serviceNames) {
- service.add(serviceName);
- }
- loadedServices.put(user, service);
- }
- }
-
- class TestServiceClient extends ServiceClient {
- @Override
- protected void serviceStart() throws Exception {
- // do nothing
- }
-
- @Override
- protected void serviceStop() throws Exception {
- // do nothing
- }
-
- @Override
- protected void serviceInit(Configuration configuration)
- throws Exception {
- // do nothing
- }
-
- @Override
- public ApplicationId actionCreate(Service service)
- throws YarnException, IOException {
- String userName =
- UserGroupInformation.getCurrentUser().getShortUserName();
- Set<String> services = submittedServices.get(userName);
- if (services == null) {
- services = new HashSet<>();
- submittedServices.put(userName, services);
- }
- if (services.contains(service.getName())) {
- String message = "Failed to create service " + service.getName()
- + ", because it already exists.";
- throw new YarnException(message);
- }
- services.add(service.getName());
- return ApplicationId.newInstance(System.currentTimeMillis(), 1);
- }
- }
-
- private void verifyForLaunchedUserServices() {
- Assert.assertEquals(loadedServices.size(), submittedServices.size());
- for (Map.Entry<String, Set<String>> entry : submittedServices.entrySet()) {
- String user = entry.getKey();
- Set<String> serviceSet = entry.getValue();
- Assert.assertTrue(loadedServices.containsKey(user));
- Set<String> services = loadedServices.get(user);
- Assert.assertEquals(services.size(), serviceSet.size());
- Assert.assertTrue(services.containsAll(serviceSet));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/example-app.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/example-app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/example-app.json
deleted file mode 100644
index a2f41cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/example-app.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties
deleted file mode 100644
index 81a3f6a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=info,stdout
-log4j.threshold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile
deleted file mode 100644
index 1d514d6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "bad",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile
deleted file mode 100644
index 823561d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app1",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile
deleted file mode 100644
index 823561d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app1",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json
deleted file mode 100644
index 8a3a561..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app3",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile
deleted file mode 100644
index 823561d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app1",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile
deleted file mode 100644
index d8fd1d1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app2",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..b89146a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
new file mode 100644
index 0000000..45168a9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
@@ -0,0 +1,144 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-services</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-yarn-services-api</artifactId>
+ <name>Apache Hadoop YARN Services API</name>
+ <packaging>jar</packaging>
+ <description>Hadoop YARN REST APIs for services</description>
+
+ <build>
+
+ <!-- resources are filtered for dynamic updates. This gets build info in-->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ <resource>
+ <directory>src/main/scripts/</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <!-- The configuration of the plugin -->
+ <configuration>
+ <!-- Configuration of the archiver -->
+ <archive>
+ <manifestEntries>
+ <mode>development</mode>
+ <url>${project.url}</url>
+ </manifestEntries>
+ <!-- Manifest specific configuration -->
+ <manifest>
+ </manifest>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/*.yarnfile</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <reporting>
+ </reporting>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- Test dependencies -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
new file mode 100644
index 0000000..a8e2f51
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -0,0 +1,598 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client;
+
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.List;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.client.api.AppAdminClient;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ComponentState;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
+import org.apache.hadoop.yarn.service.conf.RestApiConstants;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.util.RMHAUtils;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.eclipse.jetty.util.UrlEncoded;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
+
+/**
+ * The rest API client for users to manage services on YARN.
+ */
+public class ApiServiceClient extends AppAdminClient {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApiServiceClient.class);
+ protected YarnClient yarnClient;
+
+ @Override protected void serviceInit(Configuration configuration)
+ throws Exception {
+ yarnClient = YarnClient.createYarnClient();
+ addService(yarnClient);
+ super.serviceInit(configuration);
+ }
+
+ /**
+ * Calculate Resource Manager address base on working REST API.
+ */
+ private String getRMWebAddress() {
+ Configuration conf = getConfig();
+ String scheme = "http://";
+ String path = "/app/v1/services/version";
+ String rmAddress = conf
+ .get("yarn.resourcemanager.webapp.address");
+ if (YarnConfiguration.useHttps(conf)) {
+ scheme = "https://";
+ rmAddress = conf
+ .get("yarn.resourcemanager.webapp.https.address");
+ }
+ boolean useKerberos = UserGroupInformation.isSecurityEnabled();
+ List<String> rmServers = RMHAUtils
+ .getRMHAWebappAddresses(new YarnConfiguration(conf));
+ for (String host : rmServers) {
+ try {
+ Client client = Client.create();
+ StringBuilder sb = new StringBuilder();
+ sb.append(scheme);
+ sb.append(host);
+ sb.append(path);
+ if (!useKerberos) {
+ try {
+ String username = UserGroupInformation.getCurrentUser().getShortUserName();
+ sb.append("?user.name=");
+ sb.append(username);
+ } catch (IOException e) {
+ LOG.debug("Fail to resolve username: {}", e);
+ }
+ }
+ WebResource webResource = client
+ .resource(sb.toString());
+ if (useKerberos) {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ webResource.header("WWW-Authenticate", token);
+ }
+ ClientResponse test = webResource.get(ClientResponse.class);
+ if (test.getStatus() == 200) {
+ rmAddress = host;
+ break;
+ }
+ } catch (Exception e) {
+ LOG.debug("Fail to connect to: "+host, e);
+ }
+ }
+ return scheme+rmAddress;
+ }
+
+ /**
+ * Compute active resource manager API service location.
+ *
+ * @param appName - YARN service name
+ * @return URI to API Service
+ * @throws IOException
+ */
+ private String getServicePath(String appName) throws IOException {
+ String url = getRMWebAddress();
+ StringBuilder api = new StringBuilder();
+ api.append(url);
+ api.append("/app/v1/services");
+ if (appName != null) {
+ api.append("/");
+ api.append(appName);
+ }
+ Configuration conf = getConfig();
+ if (conf.get("hadoop.http.authentication.type").equalsIgnoreCase("simple")) {
+ api.append("?user.name=" + UrlEncoded
+ .encodeString(System.getProperty("user.name")));
+ }
+ return api.toString();
+ }
+
+ private String getInstancesPath(String appName) throws IOException {
+ Preconditions.checkNotNull(appName);
+ String url = getRMWebAddress();
+ StringBuilder api = new StringBuilder();
+ api.append(url);
+ api.append("/app/v1/services/").append(appName).append("/")
+ .append(RestApiConstants.COMP_INSTANCES);
+ Configuration conf = getConfig();
+ if (conf.get("hadoop.http.authentication.type").equalsIgnoreCase(
+ "simple")) {
+ api.append("?user.name=" + UrlEncoded
+ .encodeString(System.getProperty("user.name")));
+ }
+ return api.toString();
+ }
+
+ private String getComponentsPath(String appName) throws IOException {
+ Preconditions.checkNotNull(appName);
+ String url = getRMWebAddress();
+ StringBuilder api = new StringBuilder();
+ api.append(url);
+ api.append("/app/v1/services/").append(appName).append("/")
+ .append(RestApiConstants.COMPONENTS);
+ Configuration conf = getConfig();
+ if (conf.get("hadoop.http.authentication.type").equalsIgnoreCase(
+ "simple")) {
+ api.append("?user.name=" + UrlEncoded
+ .encodeString(System.getProperty("user.name")));
+ }
+ return api.toString();
+ }
+
+ private Builder getApiClient() throws IOException {
+ return getApiClient(getServicePath(null));
+ }
+
+ /**
+ * Setup API service web request.
+ *
+ * @param requestPath
+ * @return
+ * @throws IOException
+ */
+ private Builder getApiClient(String requestPath)
+ throws IOException {
+ Client client = Client.create(getClientConfig());
+ Configuration conf = getConfig();
+ client.setChunkedEncodingSize(null);
+ Builder builder = client
+ .resource(requestPath).type(MediaType.APPLICATION_JSON);
+ if (conf.get("hadoop.http.authentication.type").equals("kerberos")) {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ builder.header("WWW-Authenticate", token);
+ }
+ return builder
+ .accept("application/json;charset=utf-8");
+ }
+
+ private ClientConfig getClientConfig() {
+ ClientConfig config = new DefaultClientConfig();
+ config.getProperties().put(
+ ClientConfig.PROPERTY_CHUNKED_ENCODING_SIZE, 0);
+ config.getProperties().put(
+ ClientConfig.PROPERTY_BUFFER_RESPONSE_ENTITY_ON_EXCEPTION, true);
+ return config;
+ }
+
+ private int processResponse(ClientResponse response) {
+ response.bufferEntity();
+ String output;
+ if (response.getStatus() == 401) {
+ LOG.error("Authentication required");
+ return EXIT_EXCEPTION_THROWN;
+ }
+ if (response.getStatus() == 503) {
+ LOG.error("YARN Service is unavailable or disabled.");
+ return EXIT_EXCEPTION_THROWN;
+ }
+ try {
+ ServiceStatus ss = response.getEntity(ServiceStatus.class);
+ output = ss.getDiagnostics();
+ } catch (Throwable t) {
+ output = response.getEntity(String.class);
+ }
+ if (output==null) {
+ output = response.getEntity(String.class);
+ }
+ if (response.getStatus() <= 299) {
+ LOG.info(output);
+ return EXIT_SUCCESS;
+ } else {
+ LOG.error(output);
+ return EXIT_EXCEPTION_THROWN;
+ }
+ }
+
+ /**
+ * Utility method to load Service json from disk or from
+ * YARN examples.
+ *
+ * @param fileName - path to yarnfile
+ * @param serviceName - YARN Service Name
+ * @param lifetime - application lifetime
+ * @param queue - Queue to submit application
+ * @return
+ * @throws IOException
+ * @throws YarnException
+ */
+ public Service loadAppJsonFromLocalFS(String fileName, String serviceName,
+ Long lifetime, String queue) throws IOException, YarnException {
+ File file = new File(fileName);
+ if (!file.exists() && fileName.equals(file.getName())) {
+ String examplesDirStr = System.getenv("YARN_SERVICE_EXAMPLES_DIR");
+ String[] examplesDirs;
+ if (examplesDirStr == null) {
+ String yarnHome = System
+ .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
+ examplesDirs = new String[]{
+ yarnHome + "/share/hadoop/yarn/yarn-service-examples",
+ yarnHome + "/yarn-service-examples"
+ };
+ } else {
+ examplesDirs = StringUtils.split(examplesDirStr, ":");
+ }
+ for (String dir : examplesDirs) {
+ file = new File(MessageFormat.format("{0}/{1}/{2}.json",
+ dir, fileName, fileName));
+ if (file.exists()) {
+ break;
+ }
+ // Then look for secondary location.
+ file = new File(MessageFormat.format("{0}/{1}.json",
+ dir, fileName));
+ if (file.exists()) {
+ break;
+ }
+ }
+ }
+ if (!file.exists()) {
+ throw new YarnException("File or example could not be found: " +
+ fileName);
+ }
+ Path filePath = new Path(file.getAbsolutePath());
+ LOG.info("Loading service definition from local FS: " + filePath);
+ Service service = jsonSerDeser
+ .load(FileSystem.getLocal(getConfig()), filePath);
+ if (!StringUtils.isEmpty(serviceName)) {
+ service.setName(serviceName);
+ }
+ if (lifetime != null && lifetime > 0) {
+ service.setLifetime(lifetime);
+ }
+ if (!StringUtils.isEmpty(queue)) {
+ service.setQueue(queue);
+ }
+ return service;
+ }
+
+ /**
+ * Launch YARN service application.
+ *
+ * @param fileName - path to yarnfile
+ * @param appName - YARN Service Name
+ * @param lifetime - application lifetime
+ * @param queue - Queue to submit application
+ */
+ @Override
+ public int actionLaunch(String fileName, String appName, Long lifetime,
+ String queue) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service =
+ loadAppJsonFromLocalFS(fileName, appName, lifetime, queue);
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient()
+ .post(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to launch application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Stop YARN service application.
+ *
+ * @param appName - YARN Service Name
+ */
+ @Override
+ public int actionStop(String appName) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service = new Service();
+ service.setName(appName);
+ service.setState(ServiceState.STOPPED);
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient(getServicePath(appName))
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to stop application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Start YARN service application.
+ *
+ * @param appName - YARN Service Name
+ */
+ @Override
+ public int actionStart(String appName) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service = new Service();
+ service.setName(appName);
+ service.setState(ServiceState.STARTED);
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient(getServicePath(appName))
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to start application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Save Service configuration.
+ *
+ * @param fileName - path to Yarnfile
+ * @param appName - YARN Service Name
+ * @param lifetime - container life time
+ * @param queue - Queue to submit the application
+ */
+ @Override
+ public int actionSave(String fileName, String appName, Long lifetime,
+ String queue) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service =
+ loadAppJsonFromLocalFS(fileName, appName, lifetime, queue);
+ service.setState(ServiceState.STOPPED);
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient()
+ .post(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to save application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Decommission a YARN service.
+ *
+ * @param appName - YARN Service Name
+ */
+ @Override
+ public int actionDestroy(String appName) throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ ClientResponse response = getApiClient(getServicePath(appName))
+ .delete(ClientResponse.class);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to destroy application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ /**
+ * Change number of containers associated with a service.
+ *
+ * @param appName - YARN Service Name
+ * @param componentCounts - list of components and desired container count
+ */
+ @Override
+ public int actionFlex(String appName, Map<String, String> componentCounts)
+ throws IOException, YarnException {
+ int result = EXIT_SUCCESS;
+ try {
+ Service service = new Service();
+ service.setName(appName);
+ service.setState(ServiceState.FLEX);
+ for (Map.Entry<String, String> entry : componentCounts.entrySet()) {
+ Component component = new Component();
+ component.setName(entry.getKey());
+ Long numberOfContainers = Long.parseLong(entry.getValue());
+ component.setNumberOfContainers(numberOfContainers);
+ service.addComponent(component);
+ }
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient(getServicePath(appName))
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Fail to flex application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ @Override
+ public int enableFastLaunch(String destinationFolder) throws IOException, YarnException {
+ ServiceClient sc = new ServiceClient();
+ sc.init(getConfig());
+ sc.start();
+ int result = sc.enableFastLaunch(destinationFolder);
+ sc.close();
+ return result;
+ }
+
+ /**
+ * Retrieve Service Status through REST API.
+ *
+ * @param appIdOrName - YARN application ID or application name
+ * @return Status output
+ */
+ @Override
+ public String getStatusString(String appIdOrName) throws IOException,
+ YarnException {
+ String output = "";
+ String appName;
+ try {
+ ApplicationId appId = ApplicationId.fromString(appIdOrName);
+ ApplicationReport appReport = yarnClient.getApplicationReport(appId);
+ appName = appReport.getName();
+ } catch (IllegalArgumentException e) {
+ // not app Id format, may be app name
+ appName = appIdOrName;
+ ServiceApiUtil.validateNameFormat(appName, getConfig());
+ }
+ try {
+ ClientResponse response = getApiClient(getServicePath(appName))
+ .get(ClientResponse.class);
+ if (response.getStatus() == 404) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" Service ");
+ sb.append(appName);
+ sb.append(" not found");
+ return sb.toString();
+ }
+ if (response.getStatus() != 200) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(appName);
+ sb.append(" Failed : HTTP error code : ");
+ sb.append(response.getStatus());
+ return sb.toString();
+ }
+ output = response.getEntity(String.class);
+ } catch (Exception e) {
+ LOG.error("Fail to check application status: ", e);
+ }
+ return output;
+ }
+
+ @Override
+ public int initiateUpgrade(String appName,
+ String fileName, boolean autoFinalize) throws IOException, YarnException {
+ int result;
+ try {
+ Service service =
+ loadAppJsonFromLocalFS(fileName, appName, null, null);
+ if (autoFinalize) {
+ service.setState(ServiceState.UPGRADING_AUTO_FINALIZE);
+ } else {
+ service.setState(ServiceState.UPGRADING);
+ }
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient(getServicePath(appName))
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Failed to upgrade application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ @Override
+ public int actionUpgradeInstances(String appName, List<String> compInstances)
+ throws IOException, YarnException {
+ int result;
+ Container[] toUpgrade = new Container[compInstances.size()];
+ try {
+ int idx = 0;
+ for (String instanceName : compInstances) {
+ Container container = new Container();
+ container.setComponentInstanceName(instanceName);
+ container.setState(ContainerState.UPGRADING);
+ toUpgrade[idx++] = container;
+ }
+ String buffer = CONTAINER_JSON_SERDE.toJson(toUpgrade);
+ ClientResponse response = getApiClient(getInstancesPath(appName))
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Failed to upgrade component instance: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ @Override
+ public int actionUpgradeComponents(String appName, List<String> components)
+ throws IOException, YarnException {
+ int result;
+ Component[] toUpgrade = new Component[components.size()];
+ try {
+ int idx = 0;
+ for (String compName : components) {
+ Component component = new Component();
+ component.setName(compName);
+ component.setState(ComponentState.UPGRADING);
+ toUpgrade[idx++] = component;
+ }
+ String buffer = COMP_JSON_SERDE.toJson(toUpgrade);
+ ClientResponse response = getApiClient(getComponentsPath(appName))
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Failed to upgrade components: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
+
+ private static final JsonSerDeser<Container[]> CONTAINER_JSON_SERDE =
+ new JsonSerDeser<>(Container[].class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+
+ private static final JsonSerDeser<Component[]> COMP_JSON_SERDE =
+ new JsonSerDeser<>(Component[].class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/6] hadoop git commit: YARN-7530. Refactored YARN service API
project location. Contributed by Chandni Singh
Posted by ey...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
new file mode 100644
index 0000000..d90ae06
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -0,0 +1,594 @@
+# Hadoop YARN REST APIs for services v1 spec in YAML
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+swagger: '2.0'
+info:
+ title: "YARN Simplified API layer for services"
+ description: |
+ Bringing a new service on YARN today is not a simple experience. The APIs of existing
+ frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs)
+ or writing a complex spec (for declarative frameworks).
+
+ This simplified REST API can be used to create and manage the lifecycle of YARN services.
+ In most cases, the application owner will not be forced to make any changes to their applications.
+ This is primarily true if the application is packaged with containerization technologies like Docker.
+
+ This document describes the API specifications (aka. YarnFile) for deploying/managing
+ containerized services on YARN. The same JSON spec can be used for both REST API
+ and CLI to manage the services.
+
+ version: "1.0.0"
+ license:
+ name: Apache 2.0
+ url: http://www.apache.org/licenses/LICENSE-2.0.html
+# the domain of the service
+host: localhost
+port: 8088(default)
+# array of all schemes that your API supports
+schemes:
+ - http
+consumes:
+ - application/json
+produces:
+ - application/json
+paths:
+ /app/v1/services/version:
+ get:
+ summary: Get current version of the API server.
+ description: Get current version of the API server.
+ responses:
+ 200:
+ description: Successful request
+
+ /app/v1/services:
+ get:
+ summary: (TBD) List of services running in the cluster.
+ description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
+ responses:
+ 200:
+ description: An array of services
+ schema:
+ type: array
+ items:
+ $ref: '#/definitions/Service'
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
+ post:
+ summary: Create a service
+ description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
+ parameters:
+ - name: Service
+ in: body
+ description: Service request object
+ required: true
+ schema:
+ $ref: '#/definitions/Service'
+ responses:
+ 202:
+ description: The request to create a service is accepted
+ 400:
+ description: Invalid service definition provided in the request body
+ 500:
+ description: Failed to create a service
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
+
+ /app/v1/services/{service_name}:
+ put:
+ summary: Update a service or upgrade the binary version of the components of a running service
+ description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service.
+ The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
+ parameters:
+ - name: service_name
+ in: path
+ description: Service name
+ required: true
+ type: string
+ - name: Service
+ in: body
+ description: The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service
+ required: true
+ schema:
+ $ref: '#/definitions/Service'
+ responses:
+ 204:
+ description: Update or upgrade was successful
+ 404:
+ description: Service does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
+ delete:
+ summary: Destroy a service
+ description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
+ parameters:
+ - name: service_name
+ in: path
+ description: Service name
+ required: true
+ type: string
+ responses:
+ 204:
+ description: Destroy was successful
+ 404:
+ description: Service does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
+ get:
+ summary: Get details of a service.
+ description: Return the details (including containers) of a running service
+ parameters:
+ - name: service_name
+ in: path
+ description: Service name
+ required: true
+ type: string
+ responses:
+ 200:
+ description: a service object
+ schema:
+ type: object
+ items:
+ $ref: '#/definitions/Service'
+ examples:
+ service_name: logsearch
+ artifact:
+ id: logsearch:latest
+ type: docker
+ 404:
+ description: Service does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
+ /app/v1/services/{service_name}/components/{component_name}:
+ put:
+ summary: Flex a component's number of instances.
+ description: Set a component's desired number of instanes
+ parameters:
+ - name: service_name
+ in: path
+ description: Service name
+ required: true
+ type: string
+ - name: component_name
+ in: path
+ description: Component name
+ required: true
+ type: string
+ - name: Component
+ in: body
+ description: The definition of a component which contains the updated number of instances.
+ required: true
+ schema:
+ $ref: '#/definitions/Component'
+ responses:
+ 200:
+ description: Flex was successful
+ 404:
+ description: Service does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ServiceStatus'
+definitions:
+ Service:
+ description: a service resource has the following attributes.
+ required:
+ - name
+ - version
+ properties:
+ name:
+ type: string
+ description: A unique service name. If Registry DNS is enabled, the max length is 63 characters.
+ version:
+ type: string
+ description: Version of the service.
+ description:
+ type: string
+ description: Description of the service.
+ id:
+ type: string
+ description: A unique service id.
+ artifact:
+ description: The default artifact for all components of the service except the components which has Artifact type set to SERVICE (optional).
+ $ref: '#/definitions/Artifact'
+ resource:
+ description: The default resource for all components of the service (optional).
+ $ref: '#/definitions/Resource'
+ launch_time:
+ type: string
+ format: date
+ description: The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
+ number_of_running_containers:
+ type: integer
+ format: int64
+ description: In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.
+ lifetime:
+ type: integer
+ format: int64
+ description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
+ components:
+ description: Components of a service.
+ type: array
+ items:
+ $ref: '#/definitions/Component'
+ configuration:
+ description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.
+ $ref: '#/definitions/Configuration'
+ state:
+ description: State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state.
+ $ref: '#/definitions/ServiceState'
+ quicklinks:
+ type: object
+ description: A blob of key-value pairs of quicklinks to be exported for a service.
+ additionalProperties:
+ type: string
+ queue:
+ type: string
+ description: The YARN queue that this service should be submitted to.
+ kerberos_principal:
+ description: The principal info of the user who launches the service.
+ $ref: '#/definitions/KerberosPrincipal'
+ docker_client_config:
+ type: string
+ description: URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json).
+ ResourceInformation:
+ description:
+ ResourceInformation determines unit/value of resource types in addition to memory and vcores. It will be part of Resource object.
+ properties:
+ value:
+ type: integer
+ format: int64
+ description: Integer value of the resource.
+ unit:
+ type: string
+ description: Unit of the resource, acceptable values are - p/n/u/m/k/M/G/T/P/Ki/Mi/Gi/Ti/Pi. By default it is empty means no unit.
+ Resource:
+ description:
+ Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
+ properties:
+ profile:
+ type: string
+ description: Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.
+ cpus:
+ type: integer
+ format: int32
+ description: Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).
+ memory:
+ type: string
+ description: Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.
+ additional:
+ type: object
+ additionalProperties:
+ $ref: '#/definitions/ResourceInformation'
+ description: A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below.
+ PlacementPolicy:
+ description: Advanced placement policy of the components of a service.
+ required:
+ - constraints
+ properties:
+ constraints:
+ description: Placement constraint details.
+ type: array
+ items:
+ $ref: '#/definitions/PlacementConstraint'
+ PlacementConstraint:
+ description: Placement constraint details.
+ required:
+ - type
+ - scope
+ properties:
+ name:
+ description: An optional name associated to this constraint.
+ type: string
+ example: C1
+ type:
+ description: The type of placement.
+ $ref: '#/definitions/PlacementType'
+ scope:
+ description: The scope of placement.
+ $ref: '#/definitions/PlacementScope'
+ target_tags:
+ description: The name of the components that this component's placement policy is depending upon are added as target tags. So for affinity say, this component's containers are requesting to be placed on hosts where containers of the target tag component(s) are running on. Target tags can also contain the name of this component, in which case it implies that for anti-affinity say, no more than one container of this component can be placed on a host. Similarly, for cardinality, it would mean that containers of this component is requesting to be placed on hosts where at least minCardinality but no more than maxCardinality containers of the target tag component(s) are running.
+ type: array
+ items:
+ type: string
+ node_attributes:
+ description: Node attributes are a set of key:value(s) pairs associated with nodes.
+ type: object
+ additionalProperties:
+ type: array
+ items:
+ type: string
+ node_partitions:
+ description: Node partitions where the containers of this component can run.
+ type: array
+ items:
+ type: string
+ min_cardinality:
+ type: integer
+ format: int64
+ description: When placement type is cardinality, the minimum number of containers of the depending component that a host should have, where containers of this component can be allocated on.
+ example: 2
+ max_cardinality:
+ type: integer
+ format: int64
+ description: When placement type is cardinality, the maximum number of containers of the depending component that a host should have, where containers of this component can be allocated on.
+ example: 3
+ PlacementType:
+ description: The type of placement - affinity/anti-affinity/affinity-with-cardinality with containers of another component or containers of the same component (self).
+ properties:
+ type:
+ type: string
+ enum:
+ - AFFINITY
+ - ANTI_AFFINITY
+ - AFFINITY_WITH_CARDINALITY
+ PlacementScope:
+ description: The scope of placement for the containers of a component.
+ properties:
+ type:
+ type: string
+ enum:
+ - NODE
+ - RACK
+ Artifact:
+ description: Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
+ required:
+ - id
+ properties:
+ id:
+ type: string
+ description: Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.
+ type:
+ type: string
+ description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).
+ enum:
+ - DOCKER
+ - TARBALL
+ - SERVICE
+ default: DOCKER
+ uri:
+ type: string
+ description: Artifact location to support multiple artifact stores (optional).
+ Component:
+ description: One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
+ required:
+ - name
+ properties:
+ name:
+ type: string
+ description: Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
+ state:
+ description: The state of the component
+ $ref: "#/definitions/ComponentState"
+ dependencies:
+ type: array
+ items:
+ type: string
+ description: An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.
+ readiness_check:
+ description: Readiness check for this component.
+ $ref: '#/definitions/ReadinessCheck'
+ artifact:
+ description: Artifact of the component (optional). If not specified, the service level global artifact takes effect.
+ $ref: '#/definitions/Artifact'
+ launch_command:
+ type: string
+ description: The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).
+ resource:
+ description: Resource of this component (optional). If not specified, the service level global resource takes effect.
+ $ref: '#/definitions/Resource'
+ number_of_containers:
+ type: integer
+ format: int64
+ description: Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.
+ containers:
+ type: array
+ description: Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.
+ items:
+ $ref: '#/definitions/Container'
+ run_privileged_container:
+ type: boolean
+ description: Run all containers of this component in privileged mode (YARN-4262).
+ placement_policy:
+ description: Advanced scheduling and placement policies for all containers of this component.
+ $ref: '#/definitions/PlacementPolicy'
+ configuration:
+ description: Config properties for this component.
+ $ref: '#/definitions/Configuration'
+ quicklinks:
+ type: array
+ items:
+ type: string
+ description: A list of quicklink keys defined at the service level, and to be resolved by this component.
+ restartPolicy:
+ type: string
+ description: Policy of restart component. Including ALWAYS (Always restart component even if instance exit code = 0); ON_FAILURE (Only restart component if instance exit code != 0); NEVER (Do not restart in any cases)
+ enum:
+ - ALWAYS
+ - ON_FAILURE
+ - NEVER
+ default: ALWAYS
+ ReadinessCheck:
+ description: A check to be performed to determine the readiness of a component instance (a container). If no readiness check is specified, the default readiness check will be used unless the yarn.service.default-readiness-check.enabled configuration property is set to false at the component, service, or system level. The artifact field is currently unsupported but may be implemented in the future, enabling a pluggable helper container to support advanced use cases.
+ required:
+ - type
+ properties:
+ type:
+ type: string
+ description: DEFAULT (AM checks whether the container has an IP and optionally performs a DNS lookup for the container hostname), HTTP (AM performs default checks, plus sends a REST call to the container and expects a response code between 200 and 299), or PORT (AM performs default checks, plus attempts to open a socket connection to the container on a specified port).
+ enum:
+ - DEFAULT
+ - HTTP
+ - PORT
+ properties:
+ type: object
+ description: A blob of key value pairs that will be used to configure the check.
+ additionalProperties:
+ type: string
+ artifact:
+ description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET
+ $ref: '#/definitions/Artifact'
+ Configuration:
+ description: Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
+ properties:
+ properties:
+ type: object
+ description: A blob of key-value pairs for configuring the YARN service AM.
+ additionalProperties:
+ type: string
+ env:
+ type: object
+ description: A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.
+ additionalProperties:
+ type: string
+ files:
+ description: Array of list of files that needs to be created and made available as volumes in the service component containers.
+ type: array
+ items:
+ $ref: '#/definitions/ConfigFile'
+ ConfigFile:
+ description: A config file that needs to be created and made available as a volume in a service component container.
+ properties:
+ type:
+ type: string
+ description: Config file in the standard format like xml, properties, json, yaml, template.
+ enum:
+ - XML
+ - PROPERTIES
+ - JSON
+ - YAML
+ - TEMPLATE
+ - HADOOP_XML
+ - STATIC
+ - ARCHIVE
+ dest_file:
+ type: string
+ description: The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.
+ src_file:
+ type: string
+ description: This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.
+ properties:
+ type: object
+ description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.
+ additionalProperties:
+ type: string
+ Container:
+ description: An instance of a running service container.
+ properties:
+ id:
+ type: string
+ description: Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.
+ launch_time:
+ type: string
+ format: date
+ description: The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.
+ ip:
+ type: string
+ description: IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.
+ hostname:
+ type: string
+ description: Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.
+ bare_host:
+ type: string
+ description: The bare node or host in which the container is running, e.g. cn008.example.com.
+ state:
+ description: State of the container of a service.
+ $ref: '#/definitions/ContainerState'
+ component_instance_name:
+ type: string
+ description: Name of the component instance that this container instance belongs to. Component instance name is named as $COMPONENT_NAME-i, where i is a
+ monotonically increasing integer. E.g. A componet called nginx can have multiple component instances named as nginx-0, nginx-1 etc.
+ Each component instance is backed by a container instance.
+ resource:
+ description: Resource used for this container.
+ $ref: '#/definitions/Resource'
+ artifact:
+ description: Artifact used for this container.
+ $ref: '#/definitions/Artifact'
+ privileged_container:
+ type: boolean
+ description: Container running in privileged mode or not.
+ ServiceState:
+ description: The current state of a service.
+ properties:
+ state:
+ type: string
+ description: enum of the state of the service
+ enum:
+ - ACCEPTED
+ - STARTED
+ - STABLE
+ - STOPPED
+ - FAILED
+ - FLEX
+ - UPGRADING
+ ContainerState:
+ description: The current state of the container of a service.
+ properties:
+ state:
+ type: string
+ description: enum of the state of the container
+ enum:
+ - INIT
+ - STARTED
+ - READY
+ ComponentState:
+ description: The state of the component
+ properties:
+ state:
+ type: string
+ description: enum of the state of the component
+ enum:
+ - INIT
+ - FLEXING
+ - STABLE
+ - UPGRADING
+ ServiceStatus:
+ description: The current status of a submitted service, returned as a response to the GET API.
+ properties:
+ diagnostics:
+ type: string
+ description: Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.
+ state:
+ description: Service state.
+ $ref: '#/definitions/ServiceState'
+ code:
+ type: integer
+ format: int32
+ description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.
+ KerberosPrincipal:
+ description: The kerberos principal info of the user who launches the service.
+ properties:
+ principal_name:
+ type: string
+ description: The principal name of the user who launches the service. Note that `_HOST` is required in the `principal_name` field such as `testuser/_HOST@EXAMPLE.COM` because Hadoop client validates that the server's (in this case, the AM's) principal has hostname present when communicating to the server.
+ keytab:
+ type: string
+ description: The URI of the kerberos keytab. Currently supports only files present on the bare host. URI starts with "file\://" - A path on the local host where the keytab is stored. It is assumed that admin pre-installs the keytabs on the local host before AM launches.
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/log4j-server.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
new file mode 100644
index 0000000..8c679b9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for YARN Services REST API Server
+
+# Log rotation based on size (100KB) with a max of 10 backup files
+log4j.rootLogger=INFO, restservicelog
+log4j.threshhold=ALL
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.restservicelog=org.apache.log4j.RollingFileAppender
+log4j.appender.restservicelog.layout=org.apache.log4j.PatternLayout
+log4j.appender.restservicelog.File=${REST_SERVICE_LOG_DIR}/restservice.log
+log4j.appender.restservicelog.MaxFileSize=1GB
+log4j.appender.restservicelog.MaxBackupIndex=10
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+# for debugging REST API Service
+#log4j.logger.org.apache.hadoop.yarn.services=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
+
+log4j.logger.org.mortbay.log=DEBUG
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app
new file mode 100644
index 0000000..6a077b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DON'T DELETE. REST WEBAPP RUN SCRIPT WILL STOP WORKING.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 0000000..1282c9f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<web-app xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
+ version="3.0">
+
+ <servlet>
+ <servlet-name>Jersey REST API</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>org.apache.hadoop.yarn.service.webapp,org.apache.hadoop.yarn.service.api,org.apache.hadoop.yarn.service.api.records</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name>
+ <param-value>true</param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+ <servlet-mapping>
+ <servlet-name>Jersey REST API</servlet-name>
+ <url-pattern>/*</url-pattern>
+ </servlet-mapping>
+</web-app>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
new file mode 100644
index 0000000..75b9486
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A mock version of ServiceClient - This class is design
+ * to simulate various error conditions that will happen
+ * when a consumer class calls ServiceClient.
+ */
+public class ServiceClientTest extends ServiceClient {
+
+ private Configuration conf = new Configuration();
+ private Service goodServiceStatus = buildLiveGoodService();
+ private boolean initialized;
+ private Set<String> expectedInstances = new HashSet<>();
+
+ public ServiceClientTest() {
+ super();
+ }
+
+ @Override
+ public void init(Configuration conf) {
+ if (!initialized) {
+ super.init(conf);
+ initialized = true;
+ }
+ }
+
+ @Override
+ public void stop() {
+ // This is needed for testing API Server which uses client to get status
+ // and then perform an action.
+ }
+
+ public void forceStop() {
+ expectedInstances.clear();
+ super.stop();
+ }
+
+ @Override
+ public Configuration getConfig() {
+ return conf;
+ }
+
+ @Override
+ public ApplicationId actionCreate(Service service) throws IOException {
+ ServiceApiUtil.validateAndResolveService(service,
+ new SliderFileSystem(conf), getConfig());
+ return ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ }
+
+ @Override
+ public Service getStatus(String appName) throws FileNotFoundException {
+ if ("jenkins".equals(appName)) {
+ return goodServiceStatus;
+ } else {
+ throw new FileNotFoundException("Service " + appName + " not found");
+ }
+ }
+
+ @Override
+ public int actionStart(String serviceName)
+ throws YarnException, IOException {
+ if (serviceName != null && serviceName.equals("jenkins")) {
+ return EXIT_SUCCESS;
+ } else {
+ throw new ApplicationNotFoundException("");
+ }
+ }
+
+ @Override
+ public int actionStop(String serviceName, boolean waitForAppStopped)
+ throws YarnException, IOException {
+ if (serviceName == null) {
+ throw new NullPointerException();
+ }
+ if (serviceName.equals("jenkins")) {
+ return EXIT_SUCCESS;
+ } else if (serviceName.equals("jenkins-second-stop")) {
+ return EXIT_COMMAND_ARGUMENT_ERROR;
+ } else {
+ throw new ApplicationNotFoundException("");
+ }
+ }
+
+ @Override
+ public int actionDestroy(String serviceName) {
+ if (serviceName != null) {
+ if (serviceName.equals("jenkins")) {
+ return EXIT_SUCCESS;
+ } else if (serviceName.equals("jenkins-already-stopped")) {
+ return EXIT_SUCCESS;
+ } else if (serviceName.equals("jenkins-doesn't-exist")) {
+ return EXIT_NOT_FOUND;
+ } else if (serviceName.equals("jenkins-error-cleaning-registry")) {
+ return EXIT_OTHER_FAILURE;
+ }
+ }
+ throw new IllegalArgumentException();
+ }
+
+ @Override
+ public int initiateUpgrade(Service service) throws YarnException,
+ IOException {
+ if (service.getName() != null && service.getName().equals("jenkins")) {
+ return EXIT_SUCCESS;
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ @Override
+ public int actionUpgrade(Service service, List<Container> compInstances)
+ throws IOException, YarnException {
+ if (service.getName() != null && service.getName().equals("jenkins")
+ && compInstances != null) {
+ Set<String> actualInstances = compInstances.stream().map(
+ Container::getComponentInstanceName).collect(Collectors.toSet());
+ if (actualInstances.equals(expectedInstances)) {
+ return EXIT_SUCCESS;
+ }
+ }
+ throw new IllegalArgumentException();
+ }
+
+ Service getGoodServiceStatus() {
+ return goodServiceStatus;
+ }
+
+ void setExpectedInstances(Set<String> instances) {
+ if (instances != null) {
+ expectedInstances.addAll(instances);
+ }
+ }
+
+ static Service buildGoodService() {
+ Service service = new Service();
+ service.setName("jenkins");
+ service.setVersion("v1");
+ Artifact artifact = new Artifact();
+ artifact.setType(Artifact.TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ Component c = new Component();
+ c.setName("jenkins" + i);
+ c.setNumberOfContainers(2L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ }
+ service.setComponents(components);
+ return service;
+ }
+
+ static Service buildLiveGoodService() {
+ Service service = buildGoodService();
+ Component comp = service.getComponents().iterator().next();
+ List<Container> containers = new ArrayList<>();
+ for (int i = 0; i < comp.getNumberOfContainers(); i++) {
+ Container container = new Container();
+ container.setComponentInstanceName(comp.getName() + "-" + (i + 1));
+ container.setState(ContainerState.READY);
+ containers.add(container);
+ }
+ comp.setContainers(containers);
+ return service;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
new file mode 100644
index 0000000..733b9bc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
@@ -0,0 +1,623 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import static org.junit.Assert.*;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Path;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import com.google.common.collect.Sets;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ComponentState;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
+import org.apache.hadoop.yarn.service.conf.RestApiConstants;
+import org.apache.hadoop.yarn.service.webapp.ApiServer;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test case for ApiServer REST API.
+ *
+ */
+public class TestApiServer {
+ private ApiServer apiServer;
+ private HttpServletRequest request;
+ private ServiceClientTest mockServerClient;
+
+ @Before
+ public void setup() throws Exception {
+ request = Mockito.mock(HttpServletRequest.class);
+ Mockito.when(request.getRemoteUser())
+ .thenReturn(System.getProperty("user.name"));
+ mockServerClient = new ServiceClientTest();
+ Configuration conf = new Configuration();
+ conf.set("yarn.api-service.service.client.class",
+ ServiceClientTest.class.getName());
+ apiServer = new ApiServer(conf);
+ apiServer.setServiceClient(mockServerClient);
+ }
+
+ @After
+ public void teardown() {
+ mockServerClient.forceStop();
+ }
+
+ @Test
+ public void testPathAnnotation() {
+ assertNotNull(this.apiServer.getClass().getAnnotation(Path.class));
+ assertTrue("The controller has the annotation Path",
+ this.apiServer.getClass().isAnnotationPresent(Path.class));
+ final Path path = this.apiServer.getClass()
+ .getAnnotation(Path.class);
+ assertEquals("The path has /v1 annotation", "/v1", path.value());
+ }
+
+ @Test
+ public void testGetVersion() {
+ final Response actual = apiServer.getVersion();
+ assertEquals("Version number is", Response.ok().build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testBadCreateService() {
+ Service service = new Service();
+ // Test for invalid argument
+ final Response actual = apiServer.createService(request, service);
+ assertEquals("Create service is ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testGoodCreateService() throws Exception {
+ String json = "{\"auths\": "
+ + "{\"https://index.docker.io/v1/\": "
+ + "{\"auth\": \"foobarbaz\"},"
+ + "\"registry.example.com\": "
+ + "{\"auth\": \"bazbarfoo\"}}}";
+ File dockerTmpDir = new File("target", "docker-tmp");
+ FileUtils.deleteQuietly(dockerTmpDir);
+ dockerTmpDir.mkdirs();
+ String dockerConfig = dockerTmpDir + "/config.json";
+ BufferedWriter bw = new BufferedWriter(new FileWriter(dockerConfig));
+ bw.write(json);
+ bw.close();
+ Service service = ServiceClientTest.buildGoodService();
+ final Response actual = apiServer.createService(request, service);
+ assertEquals("Create service is ",
+ Response.status(Status.ACCEPTED).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testInternalServerErrorDockerClientConfigMissingCreateService() {
+ Service service = new Service();
+ service.setName("jenkins");
+ service.setVersion("v1");
+ service.setDockerClientConfig("/does/not/exist/config.json");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(1L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.createService(request, service);
+ assertEquals("Create service is ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testBadGetService() {
+ final String serviceName = "nonexistent-jenkins";
+ final Response actual = apiServer.getService(request, serviceName);
+ assertEquals("Get service is ",
+ Response.status(Status.NOT_FOUND).build().getStatus(),
+ actual.getStatus());
+ ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
+ assertEquals("Response code don't match",
+ RestApiConstants.ERROR_CODE_APP_NAME_INVALID, serviceStatus.getCode());
+ assertEquals("Response diagnostics don't match",
+ "Service " + serviceName + " not found",
+ serviceStatus.getDiagnostics());
+ }
+
+ @Test
+ public void testBadGetService2() {
+ final Response actual = apiServer.getService(request, null);
+ assertEquals("Get service is ",
+ Response.status(Status.NOT_FOUND).build().getStatus(),
+ actual.getStatus());
+ ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
+ assertEquals("Response code don't match",
+ RestApiConstants.ERROR_CODE_APP_NAME_INVALID, serviceStatus.getCode());
+ assertEquals("Response diagnostics don't match",
+ "Service name cannot be null.", serviceStatus.getDiagnostics());
+ }
+
+ @Test
+ public void testGoodGetService() {
+ final Response actual = apiServer.getService(request, "jenkins");
+ assertEquals("Get service is ",
+ Response.status(Status.OK).build().getStatus(), actual.getStatus());
+ }
+
+ @Test
+ public void testBadDeleteService() {
+ final Response actual = apiServer.deleteService(request, "no-jenkins");
+ assertEquals("Delete service is ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testBadDeleteService2() {
+ final Response actual = apiServer.deleteService(request, null);
+ assertEquals("Delete service is ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testBadDeleteService3() {
+ final Response actual = apiServer.deleteService(request,
+ "jenkins-doesn't-exist");
+ assertEquals("Delete service is ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testBadDeleteService4() {
+ final Response actual = apiServer.deleteService(request,
+ "jenkins-error-cleaning-registry");
+ assertEquals("Delete service is ",
+ Response.status(Status.INTERNAL_SERVER_ERROR).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testGoodDeleteService() {
+ final Response actual = apiServer.deleteService(request, "jenkins");
+ assertEquals("Delete service is ",
+ Response.status(Status.OK).build().getStatus(), actual.getStatus());
+ }
+
+ @Test
+ public void testDeleteStoppedService() {
+ final Response actual = apiServer.deleteService(request,
+ "jenkins-already-stopped");
+ assertEquals("Delete service is ",
+ Response.status(Status.OK).build().getStatus(), actual.getStatus());
+ }
+
+ @Test
+ public void testDecreaseContainerAndStop() {
+ Service service = new Service();
+ service.setState(ServiceState.STOPPED);
+ service.setName("jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(0L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.updateService(request, "jenkins",
+ service);
+ assertEquals("update service is ",
+ Response.status(Status.OK).build().getStatus(), actual.getStatus());
+ }
+
+ @Test
+ public void testBadDecreaseContainerAndStop() {
+ Service service = new Service();
+ service.setState(ServiceState.STOPPED);
+ service.setName("no-jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("no-jenkins");
+ c.setNumberOfContainers(-1L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ System.out.println("before stop");
+ final Response actual = apiServer.updateService(request, "no-jenkins",
+ service);
+ assertEquals("flex service is ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testIncreaseContainersAndStart() {
+ Service service = new Service();
+ service.setState(ServiceState.STARTED);
+ service.setName("jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(2L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.updateService(request, "jenkins",
+ service);
+ assertEquals("flex service is ",
+ Response.status(Status.OK).build().getStatus(), actual.getStatus());
+ }
+
+ @Test
+ public void testBadStartServices() {
+ Service service = new Service();
+ service.setState(ServiceState.STARTED);
+ service.setName("no-jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(2L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.updateService(request, "no-jenkins",
+ service);
+ assertEquals("start service is ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testGoodStartServices() {
+ Service service = new Service();
+ service.setState(ServiceState.STARTED);
+ service.setName("jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("jenkins");
+ c.setNumberOfContainers(2L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ final Response actual = apiServer.updateService(request, "jenkins",
+ service);
+ assertEquals("start service is ",
+ Response.status(Status.OK).build().getStatus(), actual.getStatus());
+ }
+
+ @Test
+ public void testBadStopServices() {
+ Service service = new Service();
+ service.setState(ServiceState.STOPPED);
+ service.setName("no-jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("no-jenkins");
+ c.setNumberOfContainers(-1L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ System.out.println("before stop");
+ final Response actual = apiServer.updateService(request, "no-jenkins",
+ service);
+ assertEquals("stop service is ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testGoodStopServices() {
+ Service service = new Service();
+ service.setState(ServiceState.STOPPED);
+ service.setName("jenkins");
+ System.out.println("before stop");
+ final Response actual = apiServer.updateService(request, "jenkins",
+ service);
+ assertEquals("stop service is ",
+ Response.status(Status.OK).build().getStatus(), actual.getStatus());
+ }
+
+ @Test
+ public void testBadSecondStopServices() throws Exception {
+ Service service = new Service();
+ service.setState(ServiceState.STOPPED);
+ service.setName("jenkins-second-stop");
+ // simulates stop on an already stopped service
+ System.out.println("before second stop");
+ final Response actual = apiServer.updateService(request,
+ "jenkins-second-stop", service);
+ assertEquals("stop service should have thrown 400 Bad Request: ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
+ assertEquals("Stop service should have failed with service already stopped",
+ "Service jenkins-second-stop is already stopped",
+ serviceStatus.getDiagnostics());
+ }
+
+ @Test
+ public void testUpdateService() {
+ Service service = new Service();
+ service.setState(ServiceState.STARTED);
+ service.setName("no-jenkins");
+ Artifact artifact = new Artifact();
+ artifact.setType(TypeEnum.DOCKER);
+ artifact.setId("jenkins:latest");
+ Resource resource = new Resource();
+ resource.setCpus(1);
+ resource.setMemory("2048");
+ List<Component> components = new ArrayList<Component>();
+ Component c = new Component();
+ c.setName("no-jenkins");
+ c.setNumberOfContainers(-1L);
+ c.setArtifact(artifact);
+ c.setLaunchCommand("");
+ c.setResource(resource);
+ components.add(c);
+ service.setComponents(components);
+ System.out.println("before stop");
+ final Response actual = apiServer.updateService(request, "no-jenkins",
+ service);
+ assertEquals("update service is ",
+ Response.status(Status.BAD_REQUEST)
+ .build().getStatus(), actual.getStatus());
+ }
+
+ @Test
+ public void testUpdateComponent() {
+ Response actual = apiServer.updateComponent(request, "jenkins",
+ "jenkins-master", null);
+ ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
+ assertEquals("Update component should have failed with 400 bad request",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ assertEquals("Update component should have failed with no data error",
+ "No component data provided", serviceStatus.getDiagnostics());
+
+ Component comp = new Component();
+ actual = apiServer.updateComponent(request, "jenkins", "jenkins-master",
+ comp);
+ serviceStatus = (ServiceStatus) actual.getEntity();
+ assertEquals("Update component should have failed with 400 bad request",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ assertEquals("Update component should have failed with no count error",
+ "No container count provided", serviceStatus.getDiagnostics());
+
+ comp.setNumberOfContainers(-1L);
+ actual = apiServer.updateComponent(request, "jenkins", "jenkins-master",
+ comp);
+ serviceStatus = (ServiceStatus) actual.getEntity();
+ assertEquals("Update component should have failed with 400 bad request",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ assertEquals("Update component should have failed with no count error",
+ "Invalid number of containers specified -1", serviceStatus.getDiagnostics());
+
+ comp.setName("jenkins-slave");
+ comp.setNumberOfContainers(1L);
+ actual = apiServer.updateComponent(request, "jenkins", "jenkins-master",
+ comp);
+ serviceStatus = (ServiceStatus) actual.getEntity();
+ assertEquals("Update component should have failed with 400 bad request",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ assertEquals(
+ "Update component should have failed with component name mismatch "
+ + "error",
+ "Component name in the request object (jenkins-slave) does not match "
+ + "that in the URI path (jenkins-master)",
+ serviceStatus.getDiagnostics());
+ }
+
+ @Test
+ public void testInitiateUpgrade() {
+ Service goodService = ServiceClientTest.buildLiveGoodService();
+ goodService.setVersion("v2");
+ goodService.setState(ServiceState.UPGRADING);
+ final Response actual = apiServer.updateService(request,
+ goodService.getName(), goodService);
+ assertEquals("Initiate upgrade is ",
+ Response.status(Status.ACCEPTED).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testUpgradeSingleInstance() {
+ Service goodService = ServiceClientTest.buildLiveGoodService();
+ Component comp = goodService.getComponents().iterator().next();
+ Container container = comp.getContainers().iterator().next();
+ container.setState(ContainerState.UPGRADING);
+
+ // To be able to upgrade, the service needs to be in UPGRADING
+ // and container state needs to be in NEEDS_UPGRADE.
+ Service serviceStatus = mockServerClient.getGoodServiceStatus();
+ serviceStatus.setState(ServiceState.UPGRADING);
+ Container liveContainer = serviceStatus.getComponents().iterator().next()
+ .getContainers().iterator().next();
+ liveContainer.setState(ContainerState.NEEDS_UPGRADE);
+ mockServerClient.setExpectedInstances(Sets.newHashSet(
+ liveContainer.getComponentInstanceName()));
+
+ final Response actual = apiServer.updateComponentInstance(request,
+ goodService.getName(), comp.getName(),
+ container.getComponentInstanceName(), container);
+ assertEquals("Instance upgrade is ",
+ Response.status(Status.ACCEPTED).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testUpgradeMultipleInstances() {
+ Service goodService = ServiceClientTest.buildLiveGoodService();
+ Component comp = goodService.getComponents().iterator().next();
+ comp.getContainers().forEach(container ->
+ container.setState(ContainerState.UPGRADING));
+
+ // To be able to upgrade, the service needs to be in UPGRADING
+ // and container state needs to be in NEEDS_UPGRADE.
+ Service serviceStatus = mockServerClient.getGoodServiceStatus();
+ serviceStatus.setState(ServiceState.UPGRADING);
+ Set<String> expectedInstances = new HashSet<>();
+ serviceStatus.getComponents().iterator().next().getContainers().forEach(
+ container -> {
+ container.setState(ContainerState.NEEDS_UPGRADE);
+ expectedInstances.add(container.getComponentInstanceName());
+ }
+ );
+ mockServerClient.setExpectedInstances(expectedInstances);
+
+ final Response actual = apiServer.updateComponentInstances(request,
+ goodService.getName(), comp.getContainers());
+ assertEquals("Instance upgrade is ",
+ Response.status(Status.ACCEPTED).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testUpgradeComponent() {
+ Service goodService = ServiceClientTest.buildLiveGoodService();
+ Component comp = goodService.getComponents().iterator().next();
+ comp.setState(ComponentState.UPGRADING);
+
+ // To be able to upgrade, the service needs to be in UPGRADING
+ // and component state needs to be in NEEDS_UPGRADE.
+ Service serviceStatus = mockServerClient.getGoodServiceStatus();
+ serviceStatus.setState(ServiceState.UPGRADING);
+ Component liveComp = serviceStatus.getComponent(comp.getName());
+ liveComp.setState(ComponentState.NEEDS_UPGRADE);
+ Set<String> expectedInstances = new HashSet<>();
+ liveComp.getContainers().forEach(container -> {
+ expectedInstances.add(container.getComponentInstanceName());
+ container.setState(ContainerState.NEEDS_UPGRADE);
+ });
+ mockServerClient.setExpectedInstances(expectedInstances);
+
+ final Response actual = apiServer.updateComponent(request,
+ goodService.getName(), comp.getName(), comp);
+ assertEquals("Component upgrade is ",
+ Response.status(Status.ACCEPTED).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testUpgradeMultipleComps() {
+ Service goodService = ServiceClientTest.buildLiveGoodService();
+ goodService.getComponents().forEach(comp ->
+ comp.setState(ComponentState.UPGRADING));
+
+ // To be able to upgrade, the live service needs to be in UPGRADING
+ // and component states needs to be in NEEDS_UPGRADE.
+ Service serviceStatus = mockServerClient.getGoodServiceStatus();
+ serviceStatus.setState(ServiceState.UPGRADING);
+ Set<String> expectedInstances = new HashSet<>();
+ serviceStatus.getComponents().forEach(liveComp -> {
+ liveComp.setState(ComponentState.NEEDS_UPGRADE);
+ liveComp.getContainers().forEach(liveContainer -> {
+ expectedInstances.add(liveContainer.getComponentInstanceName());
+ liveContainer.setState(ContainerState.NEEDS_UPGRADE);
+ });
+ });
+ mockServerClient.setExpectedInstances(expectedInstances);
+
+ final Response actual = apiServer.updateComponents(request,
+ goodService.getName(), goodService.getComponents());
+ assertEquals("Component upgrade is ",
+ Response.status(Status.ACCEPTED).build().getStatus(),
+ actual.getStatus());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
new file mode 100644
index 0000000..6cf0880
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
+
+/**
+ * Test case for CLI to API Service.
+ *
+ */
+public class TestApiServiceClient {
+ private static ApiServiceClient asc;
+ private static ApiServiceClient badAsc;
+ private static Server server;
+
+ /**
+ * A mocked version of API Service for testing purpose.
+ *
+ */
+ @SuppressWarnings("serial")
+ public static class TestServlet extends HttpServlet {
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ System.out.println("Get was called");
+ if (req.getPathInfo() != null
+ && req.getPathInfo().contains("nonexistent-app")) {
+ resp.setStatus(HttpServletResponse.SC_NOT_FOUND);
+ } else {
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+
+ @Override
+ protected void doPut(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+
+ @Override
+ protected void doDelete(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+
+ }
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ server = new Server(8088);
+ ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(10);
+ ServletContextHandler context = new ServletContextHandler();
+ context.setContextPath("/app");
+ server.setHandler(context);
+ context.addServlet(new ServletHolder(TestServlet.class), "/*");
+ ((ServerConnector)server.getConnectors()[0]).setHost("localhost");
+ server.start();
+
+ Configuration conf = new Configuration();
+ conf.set("yarn.resourcemanager.webapp.address",
+ "localhost:8088");
+ asc = new ApiServiceClient();
+ asc.serviceInit(conf);
+
+ Configuration conf2 = new Configuration();
+ conf2.set("yarn.resourcemanager.webapp.address",
+ "localhost:8089");
+ badAsc = new ApiServiceClient();
+ badAsc.serviceInit(conf2);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ server.stop();
+ }
+
+ @Test
+ public void testLaunch() {
+ String fileName = "target/test-classes/example-app.json";
+ String appName = "example-app";
+ long lifetime = 3600L;
+ String queue = "default";
+ try {
+ int result = asc.actionLaunch(fileName, appName, lifetime, queue);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadLaunch() {
+ String fileName = "unknown_file";
+ String appName = "unknown_app";
+ long lifetime = 3600L;
+ String queue = "default";
+ try {
+ int result = badAsc.actionLaunch(fileName, appName, lifetime, queue);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testStatus() {
+ String appName = "nonexistent-app";
+ try {
+ String result = asc.getStatusString(appName);
+ assertEquals("Status reponse don't match",
+ " Service " + appName + " not found", result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testStop() {
+ String appName = "example-app";
+ try {
+ int result = asc.actionStop(appName);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadStop() {
+ String appName = "unknown_app";
+ try {
+ int result = badAsc.actionStop(appName);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testStart() {
+ String appName = "example-app";
+ try {
+ int result = asc.actionStart(appName);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadStart() {
+ String appName = "unknown_app";
+ try {
+ int result = badAsc.actionStart(appName);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testSave() {
+ String fileName = "target/test-classes/example-app.json";
+ String appName = "example-app";
+ long lifetime = 3600L;
+ String queue = "default";
+ try {
+ int result = asc.actionSave(fileName, appName, lifetime, queue);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadSave() {
+ String fileName = "unknown_file";
+ String appName = "unknown_app";
+ long lifetime = 3600L;
+ String queue = "default";
+ try {
+ int result = badAsc.actionSave(fileName, appName, lifetime, queue);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testFlex() {
+ String appName = "example-app";
+ HashMap<String, String> componentCounts = new HashMap<String, String>();
+ try {
+ int result = asc.actionFlex(appName, componentCounts);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadFlex() {
+ String appName = "unknown_app";
+ HashMap<String, String> componentCounts = new HashMap<String, String>();
+ try {
+ int result = badAsc.actionFlex(appName, componentCounts);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testDestroy() {
+ String appName = "example-app";
+ try {
+ int result = asc.actionDestroy(appName);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testBadDestroy() {
+ String appName = "unknown_app";
+ try {
+ int result = badAsc.actionDestroy(appName);
+ assertEquals(EXIT_EXCEPTION_THROWN, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testInitiateServiceUpgrade() {
+ String appName = "example-app";
+ String upgradeFileName = "target/test-classes/example-app.json";
+ try {
+ int result = asc.initiateUpgrade(appName, upgradeFileName, false);
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testInstancesUpgrade() {
+ String appName = "example-app";
+ try {
+ int result = asc.actionUpgradeInstances(appName, Lists.newArrayList(
+ "comp-1", "comp-2"));
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testComponentsUpgrade() {
+ String appName = "example-app";
+ try {
+ int result = asc.actionUpgradeComponents(appName, Lists.newArrayList(
+ "comp"));
+ assertEquals(EXIT_SUCCESS, result);
+ } catch (IOException | YarnException e) {
+ fail();
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
new file mode 100644
index 0000000..d39083d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Test class for system service manager.
+ */
+public class TestSystemServiceManagerImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestSystemServiceManagerImpl.class);
+ private SystemServiceManagerImpl systemService;
+ private Configuration conf;
+ private String resourcePath = "system-services";
+
+ private String[] users = new String[] {"user1", "user2"};
+ private static Map<String, Set<String>> loadedServices = new HashMap<>();
+ private static Map<String, Set<String>> submittedServices = new HashMap<>();
+
+ @Before
+ public void setup() {
+ File file = new File(
+ getClass().getClassLoader().getResource(resourcePath).getFile());
+ conf = new Configuration();
+ conf.set(YarnServiceConf.YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY,
+ file.getAbsolutePath());
+ systemService = new SystemServiceManagerImpl() {
+ @Override ServiceClient getServiceClient() {
+ return new TestServiceClient();
+ }
+ };
+ systemService.init(conf); // do not call explicit start
+
+ constructUserService(users[0], "example-app1");
+ constructUserService(users[1], "example-app1", "example-app2");
+ }
+
+ @After
+ public void teadDown() {
+ systemService.stop();
+ }
+
+ @Test
+ public void testSystemServiceSubmission() throws Exception {
+ systemService.start();
+
+ /* verify for ignored sevices count */
+ Map<String, Integer> ignoredUserServices =
+ systemService.getIgnoredUserServices();
+ Assert.assertEquals(1, ignoredUserServices.size());
+ Assert.assertTrue("User user1 doesn't exist.",
+ ignoredUserServices.containsKey(users[0]));
+ int count = ignoredUserServices.get(users[0]);
+ Assert.assertEquals(1, count);
+ Assert.assertEquals(1,
+ systemService.getBadFileNameExtensionSkipCounter());
+ Assert.assertEquals(1, systemService.getBadDirSkipCounter());
+
+ Map<String, Set<Service>> userServices =
+ systemService.getSyncUserServices();
+ Assert.assertEquals(loadedServices.size(), userServices.size());
+ verifyForScannedUserServices(userServices);
+
+ verifyForLaunchedUserServices();
+
+ // 2nd time launch service to handle if service exist scenario
+ systemService.launchUserService(userServices);
+ verifyForLaunchedUserServices();
+ }
+
+ private void verifyForScannedUserServices(
+ Map<String, Set<Service>> userServices) {
+ for (String user : users) {
+ Set<Service> services = userServices.get(user);
+ Set<String> serviceNames = loadedServices.get(user);
+ Assert.assertEquals(serviceNames.size(), services.size());
+ Iterator<Service> iterator = services.iterator();
+ while (iterator.hasNext()) {
+ Service next = iterator.next();
+ Assert.assertTrue(
+ "Service name doesn't exist in expected userService "
+ + serviceNames, serviceNames.contains(next.getName()));
+ }
+ }
+ }
+
+ public void constructUserService(String user, String... serviceNames) {
+ Set<String> service = loadedServices.get(user);
+ if (service == null) {
+ service = new HashSet<>();
+ for (String serviceName : serviceNames) {
+ service.add(serviceName);
+ }
+ loadedServices.put(user, service);
+ }
+ }
+
+ class TestServiceClient extends ServiceClient {
+ @Override
+ protected void serviceStart() throws Exception {
+ // do nothing
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ // do nothing
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration)
+ throws Exception {
+ // do nothing
+ }
+
+ @Override
+ public ApplicationId actionCreate(Service service)
+ throws YarnException, IOException {
+ String userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ Set<String> services = submittedServices.get(userName);
+ if (services == null) {
+ services = new HashSet<>();
+ submittedServices.put(userName, services);
+ }
+ if (services.contains(service.getName())) {
+ String message = "Failed to create service " + service.getName()
+ + ", because it already exists.";
+ throw new YarnException(message);
+ }
+ services.add(service.getName());
+ return ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ }
+ }
+
+ private void verifyForLaunchedUserServices() {
+ Assert.assertEquals(loadedServices.size(), submittedServices.size());
+ for (Map.Entry<String, Set<String>> entry : submittedServices.entrySet()) {
+ String user = entry.getKey();
+ Set<String> serviceSet = entry.getValue();
+ Assert.assertTrue(loadedServices.containsKey(user));
+ Set<String> services = loadedServices.get(user);
+ Assert.assertEquals(services.size(), serviceSet.size());
+ Assert.assertTrue(services.containsAll(serviceSet));
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[6/6] hadoop git commit: YARN-7530. Refactored YARN service API
project location. Contributed by Chandni Singh
Posted by ey...@apache.org.
YARN-7530. Refactored YARN service API project location.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a23ff8d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a23ff8d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a23ff8d8
Branch: refs/heads/trunk
Commit: a23ff8d88001ad8e4ac4c36fc1f7691d193dc1d0
Parents: 89f5911
Author: Eric Yang <ey...@apache.org>
Authored: Fri May 18 17:29:10 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Fri May 18 17:29:10 2018 -0400
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 2 +-
.../dev-support/findbugs-exclude.xml | 20 -
.../hadoop-yarn-services-api/pom.xml | 144 ----
.../yarn/service/client/ApiServiceClient.java | 598 --------------
.../client/SystemServiceManagerImpl.java | 391 ---------
.../yarn/service/client/package-info.java | 28 -
.../hadoop/yarn/service/webapp/ApiServer.java | 818 -------------------
.../yarn/service/webapp/ApiServerWebApp.java | 161 ----
.../yarn/service/webapp/package-info.java | 28 -
.../definition/YARN-Services-Examples.md | 444 ----------
...RN-Simplified-V1-API-Layer-For-Services.yaml | 594 --------------
.../src/main/resources/log4j-server.properties | 76 --
.../src/main/resources/webapps/api-server/app | 16 -
.../src/main/webapp/WEB-INF/web.xml | 36 -
.../hadoop/yarn/service/ServiceClientTest.java | 210 -----
.../hadoop/yarn/service/TestApiServer.java | 623 --------------
.../service/client/TestApiServiceClient.java | 314 -------
.../client/TestSystemServiceManagerImpl.java | 182 -----
.../src/test/resources/example-app.json | 16 -
.../src/test/resources/log4j.properties | 19 -
.../resources/system-services/bad/bad.yarnfile | 16 -
.../sync/user1/example-app1.yarnfile | 16 -
.../sync/user1/example-app2.yarnfile | 16 -
.../sync/user1/example-app3.json | 16 -
.../sync/user2/example-app1.yarnfile | 16 -
.../sync/user2/example-app2.yarnfile | 16 -
.../dev-support/findbugs-exclude.xml | 20 +
.../hadoop-yarn-services-api/pom.xml | 144 ++++
.../yarn/service/client/ApiServiceClient.java | 598 ++++++++++++++
.../client/SystemServiceManagerImpl.java | 391 +++++++++
.../yarn/service/client/package-info.java | 28 +
.../hadoop/yarn/service/webapp/ApiServer.java | 818 +++++++++++++++++++
.../yarn/service/webapp/ApiServerWebApp.java | 161 ++++
.../yarn/service/webapp/package-info.java | 28 +
.../definition/YARN-Services-Examples.md | 444 ++++++++++
...RN-Simplified-V1-API-Layer-For-Services.yaml | 594 ++++++++++++++
.../src/main/resources/log4j-server.properties | 76 ++
.../src/main/resources/webapps/api-server/app | 16 +
.../src/main/webapp/WEB-INF/web.xml | 36 +
.../hadoop/yarn/service/ServiceClientTest.java | 210 +++++
.../hadoop/yarn/service/TestApiServer.java | 623 ++++++++++++++
.../service/client/TestApiServiceClient.java | 314 +++++++
.../client/TestSystemServiceManagerImpl.java | 182 +++++
.../src/test/resources/example-app.json | 16 +
.../src/test/resources/log4j.properties | 19 +
.../resources/system-services/bad/bad.yarnfile | 16 +
.../sync/user1/example-app1.yarnfile | 16 +
.../sync/user1/example-app2.yarnfile | 16 +
.../sync/user1/example-app3.json | 16 +
.../sync/user2/example-app1.yarnfile | 16 +
.../sync/user2/example-app2.yarnfile | 16 +
.../hadoop-yarn-services/pom.xml | 1 +
.../hadoop-yarn-applications/pom.xml | 1 -
53 files changed, 4816 insertions(+), 4816 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 382c967..a2ea08c 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -105,7 +105,7 @@
</includes>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
<include>*-sources.jar</include>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
deleted file mode 100644
index b89146a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<FindBugsFilter>
-
-</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
deleted file mode 100644
index 354c9b5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ /dev/null
@@ -1,144 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-applications</artifactId>
- <version>3.2.0-SNAPSHOT</version>
- </parent>
- <artifactId>hadoop-yarn-services-api</artifactId>
- <name>Apache Hadoop YARN Services API</name>
- <packaging>jar</packaging>
- <description>Hadoop YARN REST APIs for services</description>
-
- <build>
-
- <!-- resources are filtered for dynamic updates. This gets build info in-->
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- <filtering>true</filtering>
- </resource>
- <resource>
- <directory>src/main/scripts/</directory>
- <filtering>true</filtering>
- </resource>
- </resources>
-
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <!-- The configuration of the plugin -->
- <configuration>
- <!-- Configuration of the archiver -->
- <archive>
- <manifestEntries>
- <mode>development</mode>
- <url>${project.url}</url>
- </manifestEntries>
- <!-- Manifest specific configuration -->
- <manifest>
- </manifest>
- </archive>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>**/*.yarnfile</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <reporting>
- </reporting>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-services-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-webapp</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </dependency>
- <dependency>
- <groupId>javax.ws.rs</groupId>
- <artifactId>jsr311-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
-
- <!-- ======================================================== -->
- <!-- Test dependencies -->
- <!-- ======================================================== -->
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
deleted file mode 100644
index a8e2f51..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ /dev/null
@@ -1,598 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client;
-
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.List;
-import java.util.Map;
-
-import javax.ws.rs.core.MediaType;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.client.api.AppAdminClient;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.ComponentState;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.api.records.ServiceState;
-import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
-import org.apache.hadoop.yarn.service.conf.RestApiConstants;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.util.RMHAUtils;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-import org.eclipse.jetty.util.UrlEncoded;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.WebResource.Builder;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-
-import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
-
-/**
- * The rest API client for users to manage services on YARN.
- */
-public class ApiServiceClient extends AppAdminClient {
- private static final Logger LOG =
- LoggerFactory.getLogger(ApiServiceClient.class);
- protected YarnClient yarnClient;
-
- @Override protected void serviceInit(Configuration configuration)
- throws Exception {
- yarnClient = YarnClient.createYarnClient();
- addService(yarnClient);
- super.serviceInit(configuration);
- }
-
- /**
- * Calculate Resource Manager address base on working REST API.
- */
- private String getRMWebAddress() {
- Configuration conf = getConfig();
- String scheme = "http://";
- String path = "/app/v1/services/version";
- String rmAddress = conf
- .get("yarn.resourcemanager.webapp.address");
- if (YarnConfiguration.useHttps(conf)) {
- scheme = "https://";
- rmAddress = conf
- .get("yarn.resourcemanager.webapp.https.address");
- }
- boolean useKerberos = UserGroupInformation.isSecurityEnabled();
- List<String> rmServers = RMHAUtils
- .getRMHAWebappAddresses(new YarnConfiguration(conf));
- for (String host : rmServers) {
- try {
- Client client = Client.create();
- StringBuilder sb = new StringBuilder();
- sb.append(scheme);
- sb.append(host);
- sb.append(path);
- if (!useKerberos) {
- try {
- String username = UserGroupInformation.getCurrentUser().getShortUserName();
- sb.append("?user.name=");
- sb.append(username);
- } catch (IOException e) {
- LOG.debug("Fail to resolve username: {}", e);
- }
- }
- WebResource webResource = client
- .resource(sb.toString());
- if (useKerberos) {
- AuthenticatedURL.Token token = new AuthenticatedURL.Token();
- webResource.header("WWW-Authenticate", token);
- }
- ClientResponse test = webResource.get(ClientResponse.class);
- if (test.getStatus() == 200) {
- rmAddress = host;
- break;
- }
- } catch (Exception e) {
- LOG.debug("Fail to connect to: "+host, e);
- }
- }
- return scheme+rmAddress;
- }
-
- /**
- * Compute active resource manager API service location.
- *
- * @param appName - YARN service name
- * @return URI to API Service
- * @throws IOException
- */
- private String getServicePath(String appName) throws IOException {
- String url = getRMWebAddress();
- StringBuilder api = new StringBuilder();
- api.append(url);
- api.append("/app/v1/services");
- if (appName != null) {
- api.append("/");
- api.append(appName);
- }
- Configuration conf = getConfig();
- if (conf.get("hadoop.http.authentication.type").equalsIgnoreCase("simple")) {
- api.append("?user.name=" + UrlEncoded
- .encodeString(System.getProperty("user.name")));
- }
- return api.toString();
- }
-
- private String getInstancesPath(String appName) throws IOException {
- Preconditions.checkNotNull(appName);
- String url = getRMWebAddress();
- StringBuilder api = new StringBuilder();
- api.append(url);
- api.append("/app/v1/services/").append(appName).append("/")
- .append(RestApiConstants.COMP_INSTANCES);
- Configuration conf = getConfig();
- if (conf.get("hadoop.http.authentication.type").equalsIgnoreCase(
- "simple")) {
- api.append("?user.name=" + UrlEncoded
- .encodeString(System.getProperty("user.name")));
- }
- return api.toString();
- }
-
- private String getComponentsPath(String appName) throws IOException {
- Preconditions.checkNotNull(appName);
- String url = getRMWebAddress();
- StringBuilder api = new StringBuilder();
- api.append(url);
- api.append("/app/v1/services/").append(appName).append("/")
- .append(RestApiConstants.COMPONENTS);
- Configuration conf = getConfig();
- if (conf.get("hadoop.http.authentication.type").equalsIgnoreCase(
- "simple")) {
- api.append("?user.name=" + UrlEncoded
- .encodeString(System.getProperty("user.name")));
- }
- return api.toString();
- }
-
- private Builder getApiClient() throws IOException {
- return getApiClient(getServicePath(null));
- }
-
- /**
- * Setup API service web request.
- *
- * @param requestPath
- * @return
- * @throws IOException
- */
- private Builder getApiClient(String requestPath)
- throws IOException {
- Client client = Client.create(getClientConfig());
- Configuration conf = getConfig();
- client.setChunkedEncodingSize(null);
- Builder builder = client
- .resource(requestPath).type(MediaType.APPLICATION_JSON);
- if (conf.get("hadoop.http.authentication.type").equals("kerberos")) {
- AuthenticatedURL.Token token = new AuthenticatedURL.Token();
- builder.header("WWW-Authenticate", token);
- }
- return builder
- .accept("application/json;charset=utf-8");
- }
-
- private ClientConfig getClientConfig() {
- ClientConfig config = new DefaultClientConfig();
- config.getProperties().put(
- ClientConfig.PROPERTY_CHUNKED_ENCODING_SIZE, 0);
- config.getProperties().put(
- ClientConfig.PROPERTY_BUFFER_RESPONSE_ENTITY_ON_EXCEPTION, true);
- return config;
- }
-
- private int processResponse(ClientResponse response) {
- response.bufferEntity();
- String output;
- if (response.getStatus() == 401) {
- LOG.error("Authentication required");
- return EXIT_EXCEPTION_THROWN;
- }
- if (response.getStatus() == 503) {
- LOG.error("YARN Service is unavailable or disabled.");
- return EXIT_EXCEPTION_THROWN;
- }
- try {
- ServiceStatus ss = response.getEntity(ServiceStatus.class);
- output = ss.getDiagnostics();
- } catch (Throwable t) {
- output = response.getEntity(String.class);
- }
- if (output==null) {
- output = response.getEntity(String.class);
- }
- if (response.getStatus() <= 299) {
- LOG.info(output);
- return EXIT_SUCCESS;
- } else {
- LOG.error(output);
- return EXIT_EXCEPTION_THROWN;
- }
- }
-
- /**
- * Utility method to load Service json from disk or from
- * YARN examples.
- *
- * @param fileName - path to yarnfile
- * @param serviceName - YARN Service Name
- * @param lifetime - application lifetime
- * @param queue - Queue to submit application
- * @return
- * @throws IOException
- * @throws YarnException
- */
- public Service loadAppJsonFromLocalFS(String fileName, String serviceName,
- Long lifetime, String queue) throws IOException, YarnException {
- File file = new File(fileName);
- if (!file.exists() && fileName.equals(file.getName())) {
- String examplesDirStr = System.getenv("YARN_SERVICE_EXAMPLES_DIR");
- String[] examplesDirs;
- if (examplesDirStr == null) {
- String yarnHome = System
- .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
- examplesDirs = new String[]{
- yarnHome + "/share/hadoop/yarn/yarn-service-examples",
- yarnHome + "/yarn-service-examples"
- };
- } else {
- examplesDirs = StringUtils.split(examplesDirStr, ":");
- }
- for (String dir : examplesDirs) {
- file = new File(MessageFormat.format("{0}/{1}/{2}.json",
- dir, fileName, fileName));
- if (file.exists()) {
- break;
- }
- // Then look for secondary location.
- file = new File(MessageFormat.format("{0}/{1}.json",
- dir, fileName));
- if (file.exists()) {
- break;
- }
- }
- }
- if (!file.exists()) {
- throw new YarnException("File or example could not be found: " +
- fileName);
- }
- Path filePath = new Path(file.getAbsolutePath());
- LOG.info("Loading service definition from local FS: " + filePath);
- Service service = jsonSerDeser
- .load(FileSystem.getLocal(getConfig()), filePath);
- if (!StringUtils.isEmpty(serviceName)) {
- service.setName(serviceName);
- }
- if (lifetime != null && lifetime > 0) {
- service.setLifetime(lifetime);
- }
- if (!StringUtils.isEmpty(queue)) {
- service.setQueue(queue);
- }
- return service;
- }
-
- /**
- * Launch YARN service application.
- *
- * @param fileName - path to yarnfile
- * @param appName - YARN Service Name
- * @param lifetime - application lifetime
- * @param queue - Queue to submit application
- */
- @Override
- public int actionLaunch(String fileName, String appName, Long lifetime,
- String queue) throws IOException, YarnException {
- int result = EXIT_SUCCESS;
- try {
- Service service =
- loadAppJsonFromLocalFS(fileName, appName, lifetime, queue);
- String buffer = jsonSerDeser.toJson(service);
- ClientResponse response = getApiClient()
- .post(ClientResponse.class, buffer);
- result = processResponse(response);
- } catch (Exception e) {
- LOG.error("Fail to launch application: ", e);
- result = EXIT_EXCEPTION_THROWN;
- }
- return result;
- }
-
- /**
- * Stop YARN service application.
- *
- * @param appName - YARN Service Name
- */
- @Override
- public int actionStop(String appName) throws IOException, YarnException {
- int result = EXIT_SUCCESS;
- try {
- Service service = new Service();
- service.setName(appName);
- service.setState(ServiceState.STOPPED);
- String buffer = jsonSerDeser.toJson(service);
- ClientResponse response = getApiClient(getServicePath(appName))
- .put(ClientResponse.class, buffer);
- result = processResponse(response);
- } catch (Exception e) {
- LOG.error("Fail to stop application: ", e);
- result = EXIT_EXCEPTION_THROWN;
- }
- return result;
- }
-
- /**
- * Start YARN service application.
- *
- * @param appName - YARN Service Name
- */
- @Override
- public int actionStart(String appName) throws IOException, YarnException {
- int result = EXIT_SUCCESS;
- try {
- Service service = new Service();
- service.setName(appName);
- service.setState(ServiceState.STARTED);
- String buffer = jsonSerDeser.toJson(service);
- ClientResponse response = getApiClient(getServicePath(appName))
- .put(ClientResponse.class, buffer);
- result = processResponse(response);
- } catch (Exception e) {
- LOG.error("Fail to start application: ", e);
- result = EXIT_EXCEPTION_THROWN;
- }
- return result;
- }
-
- /**
- * Save Service configuration.
- *
- * @param fileName - path to Yarnfile
- * @param appName - YARN Service Name
- * @param lifetime - container life time
- * @param queue - Queue to submit the application
- */
- @Override
- public int actionSave(String fileName, String appName, Long lifetime,
- String queue) throws IOException, YarnException {
- int result = EXIT_SUCCESS;
- try {
- Service service =
- loadAppJsonFromLocalFS(fileName, appName, lifetime, queue);
- service.setState(ServiceState.STOPPED);
- String buffer = jsonSerDeser.toJson(service);
- ClientResponse response = getApiClient()
- .post(ClientResponse.class, buffer);
- result = processResponse(response);
- } catch (Exception e) {
- LOG.error("Fail to save application: ", e);
- result = EXIT_EXCEPTION_THROWN;
- }
- return result;
- }
-
- /**
- * Decommission a YARN service.
- *
- * @param appName - YARN Service Name
- */
- @Override
- public int actionDestroy(String appName) throws IOException, YarnException {
- int result = EXIT_SUCCESS;
- try {
- ClientResponse response = getApiClient(getServicePath(appName))
- .delete(ClientResponse.class);
- result = processResponse(response);
- } catch (Exception e) {
- LOG.error("Fail to destroy application: ", e);
- result = EXIT_EXCEPTION_THROWN;
- }
- return result;
- }
-
- /**
- * Change number of containers associated with a service.
- *
- * @param appName - YARN Service Name
- * @param componentCounts - list of components and desired container count
- */
- @Override
- public int actionFlex(String appName, Map<String, String> componentCounts)
- throws IOException, YarnException {
- int result = EXIT_SUCCESS;
- try {
- Service service = new Service();
- service.setName(appName);
- service.setState(ServiceState.FLEX);
- for (Map.Entry<String, String> entry : componentCounts.entrySet()) {
- Component component = new Component();
- component.setName(entry.getKey());
- Long numberOfContainers = Long.parseLong(entry.getValue());
- component.setNumberOfContainers(numberOfContainers);
- service.addComponent(component);
- }
- String buffer = jsonSerDeser.toJson(service);
- ClientResponse response = getApiClient(getServicePath(appName))
- .put(ClientResponse.class, buffer);
- result = processResponse(response);
- } catch (Exception e) {
- LOG.error("Fail to flex application: ", e);
- result = EXIT_EXCEPTION_THROWN;
- }
- return result;
- }
-
- @Override
- public int enableFastLaunch(String destinationFolder) throws IOException, YarnException {
- ServiceClient sc = new ServiceClient();
- sc.init(getConfig());
- sc.start();
- int result = sc.enableFastLaunch(destinationFolder);
- sc.close();
- return result;
- }
-
- /**
- * Retrieve Service Status through REST API.
- *
- * @param appIdOrName - YARN application ID or application name
- * @return Status output
- */
- @Override
- public String getStatusString(String appIdOrName) throws IOException,
- YarnException {
- String output = "";
- String appName;
- try {
- ApplicationId appId = ApplicationId.fromString(appIdOrName);
- ApplicationReport appReport = yarnClient.getApplicationReport(appId);
- appName = appReport.getName();
- } catch (IllegalArgumentException e) {
- // not app Id format, may be app name
- appName = appIdOrName;
- ServiceApiUtil.validateNameFormat(appName, getConfig());
- }
- try {
- ClientResponse response = getApiClient(getServicePath(appName))
- .get(ClientResponse.class);
- if (response.getStatus() == 404) {
- StringBuilder sb = new StringBuilder();
- sb.append(" Service ");
- sb.append(appName);
- sb.append(" not found");
- return sb.toString();
- }
- if (response.getStatus() != 200) {
- StringBuilder sb = new StringBuilder();
- sb.append(appName);
- sb.append(" Failed : HTTP error code : ");
- sb.append(response.getStatus());
- return sb.toString();
- }
- output = response.getEntity(String.class);
- } catch (Exception e) {
- LOG.error("Fail to check application status: ", e);
- }
- return output;
- }
-
- @Override
- public int initiateUpgrade(String appName,
- String fileName, boolean autoFinalize) throws IOException, YarnException {
- int result;
- try {
- Service service =
- loadAppJsonFromLocalFS(fileName, appName, null, null);
- if (autoFinalize) {
- service.setState(ServiceState.UPGRADING_AUTO_FINALIZE);
- } else {
- service.setState(ServiceState.UPGRADING);
- }
- String buffer = jsonSerDeser.toJson(service);
- ClientResponse response = getApiClient(getServicePath(appName))
- .put(ClientResponse.class, buffer);
- result = processResponse(response);
- } catch (Exception e) {
- LOG.error("Failed to upgrade application: ", e);
- result = EXIT_EXCEPTION_THROWN;
- }
- return result;
- }
-
- @Override
- public int actionUpgradeInstances(String appName, List<String> compInstances)
- throws IOException, YarnException {
- int result;
- Container[] toUpgrade = new Container[compInstances.size()];
- try {
- int idx = 0;
- for (String instanceName : compInstances) {
- Container container = new Container();
- container.setComponentInstanceName(instanceName);
- container.setState(ContainerState.UPGRADING);
- toUpgrade[idx++] = container;
- }
- String buffer = CONTAINER_JSON_SERDE.toJson(toUpgrade);
- ClientResponse response = getApiClient(getInstancesPath(appName))
- .put(ClientResponse.class, buffer);
- result = processResponse(response);
- } catch (Exception e) {
- LOG.error("Failed to upgrade component instance: ", e);
- result = EXIT_EXCEPTION_THROWN;
- }
- return result;
- }
-
- @Override
- public int actionUpgradeComponents(String appName, List<String> components)
- throws IOException, YarnException {
- int result;
- Component[] toUpgrade = new Component[components.size()];
- try {
- int idx = 0;
- for (String compName : components) {
- Component component = new Component();
- component.setName(compName);
- component.setState(ComponentState.UPGRADING);
- toUpgrade[idx++] = component;
- }
- String buffer = COMP_JSON_SERDE.toJson(toUpgrade);
- ClientResponse response = getApiClient(getComponentsPath(appName))
- .put(ClientResponse.class, buffer);
- result = processResponse(response);
- } catch (Exception e) {
- LOG.error("Failed to upgrade components: ", e);
- result = EXIT_EXCEPTION_THROWN;
- }
- return result;
- }
-
- private static final JsonSerDeser<Container[]> CONTAINER_JSON_SERDE =
- new JsonSerDeser<>(Container[].class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
-
- private static final JsonSerDeser<Component[]> COMP_JSON_SERDE =
- new JsonSerDeser<>(Component[].class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
deleted file mode 100644
index f9cfa92..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
+++ /dev/null
@@ -1,391 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.service.SystemServiceManager;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.api.records.ServiceState;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.security.PrivilegedExceptionAction;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
-
-/**
- * SystemServiceManager implementation.
- * Scan for configure system service path.
- *
- * The service path structure is as follows:
- * SYSTEM_SERVICE_DIR_PATH
- * |---- sync
- * | |--- user1
- * | | |---- service1.yarnfile
- * | | |---- service2.yarnfile
- * | |--- user2
- * | | |---- service1.yarnfile
- * | | ....
- * | |
- * |---- async
- * | |--- user3
- * | | |---- service1.yarnfile
- * | | |---- service2.yarnfile
- * | |--- user4
- * | | |---- service1.yarnfile
- * | | ....
- * | |
- *
- * sync: These services are launched at the time of service start synchronously.
- * It is a blocking service start.
- * async: These services are launched in separate thread without any delay after
- * service start. Non-blocking service start.
- */
-public class SystemServiceManagerImpl extends AbstractService
- implements SystemServiceManager {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SystemServiceManagerImpl.class);
-
- private static final String YARN_FILE_SUFFIX = ".yarnfile";
- private static final String SYNC = "sync";
- private static final String ASYNC = "async";
-
- private FileSystem fs;
- private Path systemServiceDir;
- private AtomicBoolean stopExecutors = new AtomicBoolean(false);
- private Map<String, Set<Service>> syncUserServices = new HashMap<>();
- private Map<String, Set<Service>> asyncUserServices = new HashMap<>();
- private UserGroupInformation loginUGI;
- private Thread serviceLaucher;
-
- @VisibleForTesting
- private int badFileNameExtensionSkipCounter;
- @VisibleForTesting
- private Map<String, Integer> ignoredUserServices =
- new HashMap<>();
- @VisibleForTesting
- private int badDirSkipCounter;
-
- public SystemServiceManagerImpl() {
- super(SystemServiceManagerImpl.class.getName());
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- String dirPath =
- conf.get(YarnServiceConf.YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY);
- if (dirPath != null) {
- systemServiceDir = new Path(dirPath);
- LOG.info("System Service Directory is configured to {}",
- systemServiceDir);
- fs = systemServiceDir.getFileSystem(conf);
- this.loginUGI = UserGroupInformation.isSecurityEnabled() ?
- UserGroupInformation.getLoginUser() :
- UserGroupInformation.getCurrentUser();
- LOG.info("UserGroupInformation initialized to {}", loginUGI);
- }
- }
-
- @Override
- protected void serviceStart() throws Exception {
- scanForUserServices();
- launchUserService(syncUserServices);
- // Create a thread and submit services in background otherwise it
- // block RM switch time.
- serviceLaucher = new Thread(createRunnable());
- serviceLaucher.setName("System service launcher");
- serviceLaucher.start();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- LOG.info("Stopping {}", getName());
- stopExecutors.set(true);
-
- if (serviceLaucher != null) {
- serviceLaucher.interrupt();
- try {
- serviceLaucher.join();
- } catch (InterruptedException ie) {
- LOG.warn("Interrupted Exception while stopping", ie);
- }
- }
- }
-
- private Runnable createRunnable() {
- return new Runnable() {
- @Override
- public void run() {
- launchUserService(asyncUserServices);
- }
- };
- }
-
- void launchUserService(Map<String, Set<Service>> userServices) {
- for (Map.Entry<String, Set<Service>> entry : userServices.entrySet()) {
- String user = entry.getKey();
- Set<Service> services = entry.getValue();
- if (services.isEmpty()) {
- continue;
- }
- ServiceClient serviceClient = null;
- try {
- UserGroupInformation userUgi = getProxyUser(user);
- serviceClient = createServiceClient(userUgi);
- for (Service service : services) {
- LOG.info("POST: createService = {} user = {}", service, userUgi);
- try {
- launchServices(userUgi, serviceClient, service);
- } catch (IOException | UndeclaredThrowableException e) {
- if (e.getCause() != null) {
- LOG.warn(e.getCause().getMessage());
- } else {
- String message =
- "Failed to create service " + service.getName() + " : ";
- LOG.error(message, e);
- }
- }
- }
- } catch (InterruptedException e) {
- LOG.warn("System service launcher thread interrupted", e);
- break;
- } catch (Exception e) {
- LOG.error("Error while submitting services for user " + user, e);
- } finally {
- if (serviceClient != null) {
- try {
- serviceClient.close();
- } catch (IOException e) {
- LOG.warn("Error while closing serviceClient for user {}", user);
- }
- }
- }
- }
- }
-
- private ServiceClient createServiceClient(UserGroupInformation userUgi)
- throws IOException, InterruptedException {
- ServiceClient serviceClient =
- userUgi.doAs(new PrivilegedExceptionAction<ServiceClient>() {
- @Override public ServiceClient run()
- throws IOException, YarnException {
- ServiceClient sc = getServiceClient();
- sc.init(getConfig());
- sc.start();
- return sc;
- }
- });
- return serviceClient;
- }
-
- private void launchServices(UserGroupInformation userUgi,
- ServiceClient serviceClient, Service service)
- throws IOException, InterruptedException {
- if (service.getState() == ServiceState.STOPPED) {
- userUgi.doAs(new PrivilegedExceptionAction<Void>() {
- @Override public Void run() throws IOException, YarnException {
- serviceClient.actionBuild(service);
- return null;
- }
- });
- LOG.info("Service {} version {} saved.", service.getName(),
- service.getVersion());
- } else {
- ApplicationId applicationId =
- userUgi.doAs(new PrivilegedExceptionAction<ApplicationId>() {
- @Override public ApplicationId run()
- throws IOException, YarnException {
- ApplicationId applicationId = serviceClient.actionCreate(service);
- return applicationId;
- }
- });
- LOG.info("Service {} submitted with Application ID: {}",
- service.getName(), applicationId);
- }
- }
-
- ServiceClient getServiceClient() {
- return new ServiceClient();
- }
-
- private UserGroupInformation getProxyUser(String user) {
- UserGroupInformation ugi;
- if (UserGroupInformation.isSecurityEnabled()) {
- ugi = UserGroupInformation.createProxyUser(user, loginUGI);
- } else {
- ugi = UserGroupInformation.createRemoteUser(user);
- }
- return ugi;
- }
-
- // scan for both launch service types i.e sync and async
- void scanForUserServices() throws IOException {
- if (systemServiceDir == null) {
- return;
- }
- try {
- LOG.info("Scan for launch type on {}", systemServiceDir);
- RemoteIterator<FileStatus> iterLaunchType = list(systemServiceDir);
- while (iterLaunchType.hasNext()) {
- FileStatus launchType = iterLaunchType.next();
- if (!launchType.isDirectory()) {
- LOG.debug("Scanner skips for unknown file {}", launchType.getPath());
- continue;
- }
- if (launchType.getPath().getName().equals(SYNC)) {
- scanForUserServiceDefinition(launchType.getPath(), syncUserServices);
- } else if (launchType.getPath().getName().equals(ASYNC)) {
- scanForUserServiceDefinition(launchType.getPath(), asyncUserServices);
- } else {
- badDirSkipCounter++;
- LOG.debug("Scanner skips for unknown dir {}.", launchType.getPath());
- }
- }
- } catch (FileNotFoundException e) {
- LOG.warn("System service directory {} doesn't not exist.",
- systemServiceDir);
- }
- }
-
- // Files are under systemServiceDir/<users>. Scan for 2 levels
- // 1st level for users
- // 2nd level for service definitions under user
- private void scanForUserServiceDefinition(Path userDirPath,
- Map<String, Set<Service>> userServices) throws IOException {
- LOG.info("Scan for users on {}", userDirPath);
- RemoteIterator<FileStatus> iterUsers = list(userDirPath);
- while (iterUsers.hasNext()) {
- FileStatus userDir = iterUsers.next();
- // if 1st level is not user directory then skip it.
- if (!userDir.isDirectory()) {
- LOG.info(
- "Service definition {} doesn't belong to any user. Ignoring.. ",
- userDir.getPath().getName());
- continue;
- }
- String userName = userDir.getPath().getName();
- LOG.info("Scanning service definitions for user {}.", userName);
-
- //2nd level scan
- RemoteIterator<FileStatus> iterServices = list(userDir.getPath());
- while (iterServices.hasNext()) {
- FileStatus serviceCache = iterServices.next();
- String filename = serviceCache.getPath().getName();
- if (!serviceCache.isFile()) {
- LOG.info("Scanner skips for unknown dir {}", filename);
- continue;
- }
- if (!filename.endsWith(YARN_FILE_SUFFIX)) {
- LOG.info("Scanner skips for unknown file extension, filename = {}",
- filename);
- badFileNameExtensionSkipCounter++;
- continue;
- }
- Service service = getServiceDefinition(serviceCache.getPath());
- if (service != null) {
- Set<Service> services = userServices.get(userName);
- if (services == null) {
- services = new HashSet<>();
- userServices.put(userName, services);
- }
- if (!services.add(service)) {
- int count = ignoredUserServices.containsKey(userName) ?
- ignoredUserServices.get(userName) : 0;
- ignoredUserServices.put(userName, count + 1);
- LOG.warn(
- "Ignoring service {} for the user {} as it is already present,"
- + " filename = {}", service.getName(), userName, filename);
- } else {
- LOG.info("Added service {} for the user {}, filename = {}",
- service.getName(), userName, filename);
- }
- }
- }
- }
- }
-
- private Service getServiceDefinition(Path filePath) {
- Service service = null;
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Loading service definition from FS: " + filePath);
- }
- service = jsonSerDeser.load(fs, filePath);
- } catch (IOException e) {
- LOG.info("Error while loading service definition from FS: {}", e);
- }
- return service;
- }
-
- private RemoteIterator<FileStatus> list(Path path) throws IOException {
- return new StoppableRemoteIterator(fs.listStatusIterator(path));
- }
-
- @VisibleForTesting Map<String, Integer> getIgnoredUserServices() {
- return ignoredUserServices;
- }
-
- private class StoppableRemoteIterator implements RemoteIterator<FileStatus> {
- private final RemoteIterator<FileStatus> remote;
-
- StoppableRemoteIterator(RemoteIterator<FileStatus> remote) {
- this.remote = remote;
- }
-
- @Override public boolean hasNext() throws IOException {
- return !stopExecutors.get() && remote.hasNext();
- }
-
- @Override public FileStatus next() throws IOException {
- return remote.next();
- }
- }
-
- @VisibleForTesting
- Map<String, Set<Service>> getSyncUserServices() {
- return syncUserServices;
- }
-
- @VisibleForTesting
- int getBadFileNameExtensionSkipCounter() {
- return badFileNameExtensionSkipCounter;
- }
-
- @VisibleForTesting
- int getBadDirSkipCounter() {
- return badDirSkipCounter;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java
deleted file mode 100644
index cf5ce11..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.service.client contains classes
- * for YARN Services Client API.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
deleted file mode 100644
index 46c9abe..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ /dev/null
@@ -1,818 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.webapp;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.ComponentState;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.api.records.ServiceState;
-import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
-import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.conf.RestApiConstants;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED;
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
-import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
-
-/**
- * The rest API endpoints for users to manage services on YARN.
- */
-@Singleton
-@Path(CONTEXT_ROOT)
-public class ApiServer {
-
- public ApiServer() {
- super();
- }
-
- @Inject
- public ApiServer(Configuration conf) {
- super();
- }
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ApiServer.class);
- private static Configuration YARN_CONFIG = new YarnConfiguration();
- private ServiceClient serviceClientUnitTest;
- private boolean unitTest = false;
-
- static {
- init();
- }
-
- // initialize all the common resources - order is important
- private static void init() {
- }
-
- @GET
- @Path(VERSION)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
- public Response getVersion() {
- String version = VersionInfo.getBuildVersion();
- LOG.info(version);
- return Response.ok("{ \"hadoop_version\": \"" + version + "\"}").build();
- }
-
- @POST
- @Path(SERVICE_ROOT_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
- public Response createService(@Context HttpServletRequest request,
- Service service) {
- ServiceStatus serviceStatus = new ServiceStatus();
- try {
- UserGroupInformation ugi = getProxyUser(request);
- LOG.info("POST: createService = {} user = {}", service, ugi);
- if(service.getState()==ServiceState.STOPPED) {
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws YarnException, IOException {
- ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- sc.actionBuild(service);
- sc.close();
- return null;
- }
- });
- serviceStatus.setDiagnostics("Service " + service.getName() +
- " version " + service.getVersion() + " saved.");
- } else {
- ApplicationId applicationId = ugi
- .doAs(new PrivilegedExceptionAction<ApplicationId>() {
- @Override
- public ApplicationId run() throws IOException, YarnException {
- ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- ApplicationId applicationId = sc.actionCreate(service);
- sc.close();
- return applicationId;
- }
- });
- serviceStatus.setDiagnostics("Application ID: " + applicationId);
- }
- serviceStatus.setState(ACCEPTED);
- serviceStatus.setUri(
- CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
- .getName());
- return formatResponse(Status.ACCEPTED, serviceStatus);
- } catch (AccessControlException e) {
- serviceStatus.setDiagnostics(e.getMessage());
- return formatResponse(Status.FORBIDDEN, e.getCause().getMessage());
- } catch (IllegalArgumentException e) {
- return formatResponse(Status.BAD_REQUEST, e.getMessage());
- } catch (IOException | InterruptedException e) {
- String message = "Failed to create service " + service.getName()
- + ": {}";
- LOG.error(message, e);
- return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
- } catch (UndeclaredThrowableException e) {
- String message = "Failed to create service " + service.getName()
- + ": {}";
- LOG.error(message, e);
- if (e.getCause().getMessage().contains("already exists")) {
- message = "Service name " + service.getName() + " is already taken.";
- } else {
- message = e.getCause().getMessage();
- }
- return formatResponse(Status.INTERNAL_SERVER_ERROR,
- message);
- }
- }
-
- @GET
- @Path(SERVICE_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
- public Response getService(@Context HttpServletRequest request,
- @PathParam(SERVICE_NAME) String appName) {
- ServiceStatus serviceStatus = new ServiceStatus();
- try {
- if (appName == null) {
- throw new IllegalArgumentException("Service name cannot be null.");
- }
- UserGroupInformation ugi = getProxyUser(request);
- LOG.info("GET: getService for appName = {} user = {}", appName, ugi);
- Service app = getServiceFromClient(ugi, appName);
- return Response.ok(app).build();
- } catch (AccessControlException e) {
- return formatResponse(Status.FORBIDDEN, e.getMessage());
- } catch (IllegalArgumentException e) {
- serviceStatus.setDiagnostics(e.getMessage());
- serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
- return Response.status(Status.NOT_FOUND).entity(serviceStatus)
- .build();
- } catch (FileNotFoundException e) {
- serviceStatus.setDiagnostics("Service " + appName + " not found");
- serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
- return Response.status(Status.NOT_FOUND).entity(serviceStatus)
- .build();
- } catch (IOException | InterruptedException e) {
- LOG.error("Get service failed: {}", e);
- return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
- } catch (UndeclaredThrowableException e) {
- LOG.error("Get service failed: {}", e);
- return formatResponse(Status.INTERNAL_SERVER_ERROR,
- e.getCause().getMessage());
- }
- }
-
- @DELETE
- @Path(SERVICE_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
- public Response deleteService(@Context HttpServletRequest request,
- @PathParam(SERVICE_NAME) String appName) {
- try {
- if (appName == null) {
- throw new IllegalArgumentException("Service name can not be null.");
- }
- UserGroupInformation ugi = getProxyUser(request);
- LOG.info("DELETE: deleteService for appName = {} user = {}",
- appName, ugi);
- return stopService(appName, true, ugi);
- } catch (AccessControlException e) {
- return formatResponse(Status.FORBIDDEN, e.getMessage());
- } catch (IllegalArgumentException e) {
- return formatResponse(Status.BAD_REQUEST, e.getMessage());
- } catch (UndeclaredThrowableException e) {
- LOG.error("Fail to stop service: {}", e);
- return formatResponse(Status.BAD_REQUEST,
- e.getCause().getMessage());
- } catch (YarnException | FileNotFoundException e) {
- return formatResponse(Status.NOT_FOUND, e.getMessage());
- } catch (Exception e) {
- LOG.error("Fail to stop service: {}", e);
- return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
- }
- }
-
- private Response stopService(String appName, boolean destroy,
- final UserGroupInformation ugi) throws Exception {
- int result = ugi.doAs(new PrivilegedExceptionAction<Integer>() {
- @Override
- public Integer run() throws Exception {
- int result = 0;
- ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Exception stopException = null;
- try {
- result = sc.actionStop(appName, destroy);
- if (result == EXIT_SUCCESS) {
- LOG.info("Successfully stopped service {}", appName);
- }
- } catch (Exception e) {
- LOG.info("Got exception stopping service", e);
- stopException = e;
- }
- if (destroy) {
- result = sc.actionDestroy(appName);
- if (result == EXIT_SUCCESS) {
- LOG.info("Successfully deleted service {}", appName);
- }
- } else {
- if (stopException != null) {
- throw stopException;
- }
- }
- sc.close();
- return result;
- }
- });
- ServiceStatus serviceStatus = new ServiceStatus();
- if (destroy) {
- if (result == EXIT_SUCCESS) {
- serviceStatus.setDiagnostics("Successfully destroyed service " +
- appName);
- } else {
- if (result == EXIT_NOT_FOUND) {
- serviceStatus
- .setDiagnostics("Service " + appName + " doesn't exist");
- return formatResponse(Status.BAD_REQUEST, serviceStatus);
- } else {
- serviceStatus
- .setDiagnostics("Service " + appName + " error cleaning up " +
- "registry");
- return formatResponse(Status.INTERNAL_SERVER_ERROR, serviceStatus);
- }
- }
- } else {
- if (result == EXIT_COMMAND_ARGUMENT_ERROR) {
- serviceStatus
- .setDiagnostics("Service " + appName + " is already stopped");
- return formatResponse(Status.BAD_REQUEST, serviceStatus);
- } else {
- serviceStatus.setDiagnostics("Successfully stopped service " + appName);
- }
- }
- return formatResponse(Status.OK, serviceStatus);
- }
-
- @PUT
- @Path(COMPONENTS_PATH)
- @Consumes({MediaType.APPLICATION_JSON})
- @Produces({RestApiConstants.MEDIA_TYPE_JSON_UTF8, MediaType.TEXT_PLAIN})
- public Response updateComponents(@Context HttpServletRequest request,
- @PathParam(SERVICE_NAME) String serviceName,
- List<Component> requestComponents) {
-
- try {
- if (requestComponents == null || requestComponents.isEmpty()) {
- throw new YarnException("No components provided.");
- }
- UserGroupInformation ugi = getProxyUser(request);
- Set<String> compNamesToUpgrade = new HashSet<>();
- requestComponents.forEach(reqComp -> {
- if (reqComp.getState() != null &&
- reqComp.getState().equals(ComponentState.UPGRADING)) {
- compNamesToUpgrade.add(reqComp.getName());
- }
- });
- LOG.info("PUT: upgrade components {} for service {} " +
- "user = {}", compNamesToUpgrade, serviceName, ugi);
- return processComponentsUpgrade(ugi, serviceName, compNamesToUpgrade);
- } catch (AccessControlException e) {
- return formatResponse(Response.Status.FORBIDDEN, e.getMessage());
- } catch (YarnException e) {
- return formatResponse(Response.Status.BAD_REQUEST, e.getMessage());
- } catch (IOException | InterruptedException e) {
- return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
- e.getMessage());
- } catch (UndeclaredThrowableException e) {
- return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
- e.getCause().getMessage());
- }
- }
-
- @PUT
- @Path(COMPONENT_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8",
- MediaType.TEXT_PLAIN })
- public Response updateComponent(@Context HttpServletRequest request,
- @PathParam(SERVICE_NAME) String appName,
- @PathParam(COMPONENT_NAME) String componentName, Component component) {
-
- try {
- if (component == null) {
- throw new YarnException("No component data provided");
- }
- if (component.getName() != null
- && !component.getName().equals(componentName)) {
- String msg = "Component name in the request object ("
- + component.getName() + ") does not match that in the URI path ("
- + componentName + ")";
- throw new YarnException(msg);
- }
- UserGroupInformation ugi = getProxyUser(request);
- if (component.getState() != null &&
- component.getState().equals(ComponentState.UPGRADING)) {
- LOG.info("PUT: upgrade component {} for service {} " +
- "user = {}", component.getName(), appName, ugi);
- return processComponentsUpgrade(ugi, appName,
- Sets.newHashSet(componentName));
- }
-
- if (component.getNumberOfContainers() == null) {
- throw new YarnException("No container count provided");
- }
- if (component.getNumberOfContainers() < 0) {
- String message = "Invalid number of containers specified "
- + component.getNumberOfContainers();
- throw new YarnException(message);
- }
- Map<String, Long> original = ugi
- .doAs(new PrivilegedExceptionAction<Map<String, Long>>() {
- @Override
- public Map<String, Long> run() throws YarnException, IOException {
- ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Map<String, Long> original = sc.flexByRestService(appName,
- Collections.singletonMap(componentName,
- component.getNumberOfContainers()));
- sc.close();
- return original;
- }
- });
- ServiceStatus status = new ServiceStatus();
- status.setDiagnostics(
- "Updating component (" + componentName + ") size from " + original
- .get(componentName) + " to " + component.getNumberOfContainers());
- return formatResponse(Status.OK, status);
- } catch (AccessControlException e) {
- return formatResponse(Status.FORBIDDEN, e.getMessage());
- } catch (YarnException e) {
- return formatResponse(Status.BAD_REQUEST, e.getMessage());
- } catch (IOException | InterruptedException e) {
- return formatResponse(Status.INTERNAL_SERVER_ERROR,
- e.getMessage());
- } catch (UndeclaredThrowableException e) {
- return formatResponse(Status.INTERNAL_SERVER_ERROR,
- e.getCause().getMessage());
- }
- }
-
- @PUT
- @Path(SERVICE_PATH)
- @Consumes({ MediaType.APPLICATION_JSON })
- @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
- public Response updateService(@Context HttpServletRequest request,
- @PathParam(SERVICE_NAME) String appName,
- Service updateServiceData) {
- try {
- UserGroupInformation ugi = getProxyUser(request);
- LOG.info("PUT: updateService for app = {} with data = {} user = {}",
- appName, updateServiceData, ugi);
- // Ignore the app name provided in updateServiceData and always use
- // appName path param
- updateServiceData.setName(appName);
-
- if (updateServiceData.getState() != null
- && updateServiceData.getState() == ServiceState.FLEX) {
- return flexService(updateServiceData, ugi);
- }
- // For STOP the app should be running. If already stopped then this
- // operation will be a no-op. For START it should be in stopped state.
- // If already running then this operation will be a no-op.
- if (updateServiceData.getState() != null
- && updateServiceData.getState() == ServiceState.STOPPED) {
- return stopService(appName, false, ugi);
- }
-
- // If a START is requested
- if (updateServiceData.getState() != null
- && updateServiceData.getState() == ServiceState.STARTED) {
- return startService(appName, ugi);
- }
-
- // If an UPGRADE is requested
- if (updateServiceData.getState() != null && (
- updateServiceData.getState() == ServiceState.UPGRADING ||
- updateServiceData.getState() ==
- ServiceState.UPGRADING_AUTO_FINALIZE)) {
- return upgradeService(updateServiceData, ugi);
- }
-
- // If new lifetime value specified then update it
- if (updateServiceData.getLifetime() != null
- && updateServiceData.getLifetime() > 0) {
- return updateLifetime(appName, updateServiceData, ugi);
- }
- } catch (UndeclaredThrowableException e) {
- return formatResponse(Status.BAD_REQUEST,
- e.getCause().getMessage());
- } catch (AccessControlException e) {
- return formatResponse(Status.FORBIDDEN, e.getMessage());
- } catch (FileNotFoundException e) {
- String message = "Application is not found app: " + appName;
- LOG.error(message, e);
- return formatResponse(Status.NOT_FOUND, e.getMessage());
- } catch (YarnException e) {
- String message = "Service is not found in hdfs: " + appName;
- LOG.error(message, e);
- return formatResponse(Status.NOT_FOUND, e.getMessage());
- } catch (Exception e) {
- String message = "Error while performing operation for app: " + appName;
- LOG.error(message, e);
- return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
- }
-
- // If nothing happens consider it a no-op
- return Response.status(Status.NO_CONTENT).build();
- }
-
- @PUT
- @Path(COMP_INSTANCE_LONG_PATH)
- @Consumes({MediaType.APPLICATION_JSON})
- @Produces({RestApiConstants.MEDIA_TYPE_JSON_UTF8, MediaType.TEXT_PLAIN})
- public Response updateComponentInstance(@Context HttpServletRequest request,
- @PathParam(SERVICE_NAME) String serviceName,
- @PathParam(COMPONENT_NAME) String componentName,
- @PathParam(COMP_INSTANCE_NAME) String compInstanceName,
- Container reqContainer) {
-
- try {
- UserGroupInformation ugi = getProxyUser(request);
- LOG.info("PUT: update component instance {} for component = {}" +
- " service = {} user = {}", compInstanceName, componentName,
- serviceName, ugi);
- if (reqContainer == null) {
- throw new YarnException("No container data provided.");
- }
- Service service = getServiceFromClient(ugi, serviceName);
- Component component = service.getComponent(componentName);
- if (component == null) {
- throw new YarnException(String.format(
- "The component name in the URI path (%s) is invalid.",
- componentName));
- }
-
- Container liveContainer = component.getComponentInstance(
- compInstanceName);
- if (liveContainer == null) {
- throw new YarnException(String.format(
- "The component (%s) does not have a component instance (%s).",
- componentName, compInstanceName));
- }
-
- if (reqContainer.getState() != null
- && reqContainer.getState().equals(ContainerState.UPGRADING)) {
- return processContainersUpgrade(ugi, service,
- Lists.newArrayList(liveContainer));
- }
- } catch (AccessControlException e) {
- return formatResponse(Response.Status.FORBIDDEN, e.getMessage());
- } catch (YarnException e) {
- return formatResponse(Response.Status.BAD_REQUEST, e.getMessage());
- } catch (IOException | InterruptedException e) {
- return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
- e.getMessage());
- } catch (UndeclaredThrowableException e) {
- return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
- e.getCause().getMessage());
- }
- return Response.status(Status.NO_CONTENT).build();
- }
-
- @PUT
- @Path(COMP_INSTANCES_PATH)
- @Consumes({MediaType.APPLICATION_JSON})
- @Produces({RestApiConstants.MEDIA_TYPE_JSON_UTF8, MediaType.TEXT_PLAIN})
- public Response updateComponentInstances(@Context HttpServletRequest request,
- @PathParam(SERVICE_NAME) String serviceName,
- List<Container> requestContainers) {
-
- try {
- if (requestContainers == null || requestContainers.isEmpty()) {
- throw new YarnException("No containers provided.");
- }
- UserGroupInformation ugi = getProxyUser(request);
- List<String> toUpgrade = new ArrayList<>();
- for (Container reqContainer : requestContainers) {
- if (reqContainer.getState() != null &&
- reqContainer.getState().equals(ContainerState.UPGRADING)) {
- toUpgrade.add(reqContainer.getComponentInstanceName());
- }
- }
-
- if (!toUpgrade.isEmpty()) {
- Service service = getServiceFromClient(ugi, serviceName);
- LOG.info("PUT: upgrade component instances {} for service = {} " +
- "user = {}", toUpgrade, serviceName, ugi);
- List<Container> liveContainers = ServiceApiUtil
- .getLiveContainers(service, toUpgrade);
-
- return processContainersUpgrade(ugi, service, liveContainers);
- }
- } catch (AccessControlException e) {
- return formatResponse(Response.Status.FORBIDDEN, e.getMessage());
- } catch (YarnException e) {
- return formatResponse(Response.Status.BAD_REQUEST, e.getMessage());
- } catch (IOException | InterruptedException e) {
- return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
- e.getMessage());
- } catch (UndeclaredThrowableException e) {
- return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
- e.getCause().getMessage());
- }
- return Response.status(Status.NO_CONTENT).build();
- }
-
- private Response flexService(Service service, UserGroupInformation ugi)
- throws IOException, InterruptedException {
- String appName = service.getName();
- Response response = Response.status(Status.BAD_REQUEST).build();
- Map<String, String> componentCountStrings = new HashMap<String, String>();
- for (Component c : service.getComponents()) {
- componentCountStrings.put(c.getName(),
- c.getNumberOfContainers().toString());
- }
- Integer result = ugi.doAs(new PrivilegedExceptionAction<Integer>() {
-
- @Override
- public Integer run() throws YarnException, IOException {
- int result = 0;
- ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc
- .actionFlex(appName, componentCountStrings);
- sc.close();
- return Integer.valueOf(result);
- }
- });
- if (result == EXIT_SUCCESS) {
- String message = "Service " + appName + " is successfully flexed.";
- LOG.info(message);
- ServiceStatus status = new ServiceStatus();
- status.setDiagnostics(message);
- status.setState(ServiceState.ACCEPTED);
- response = formatResponse(Status.ACCEPTED, status);
- }
- return response;
- }
-
- private Response updateLifetime(String appName, Service updateAppData,
- final UserGroupInformation ugi) throws IOException,
- InterruptedException {
- String newLifeTime = ugi.doAs(new PrivilegedExceptionAction<String>() {
- @Override
- public String run() throws YarnException, IOException {
- ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- String newLifeTime = sc.updateLifetime(appName,
- updateAppData.getLifetime());
- sc.close();
- return newLifeTime;
- }
- });
- ServiceStatus status = new ServiceStatus();
- status.setDiagnostics(
- "Service (" + appName + ")'s lifeTime is updated to " + newLifeTime
- + ", " + updateAppData.getLifetime() + " seconds remaining");
- return formatResponse(Status.OK, status);
- }
-
- private Response startService(String appName,
- final UserGroupInformation ugi) throws IOException,
- InterruptedException {
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws YarnException, IOException {
- ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- sc.actionStart(appName);
- sc.close();
- return null;
- }
- });
- LOG.info("Successfully started service " + appName);
- ServiceStatus status = new ServiceStatus();
- status.setDiagnostics("Service " + appName + " is successfully started.");
- status.setState(ServiceState.ACCEPTED);
- return formatResponse(Status.OK, status);
- }
-
- private Response upgradeService(Service service,
- final UserGroupInformation ugi) throws IOException, InterruptedException {
- ServiceStatus status = new ServiceStatus();
- ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
- ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- sc.initiateUpgrade(service);
- sc.close();
- return null;
- });
- LOG.info("Service {} version {} upgrade initialized", service.getName(),
- service.getVersion());
- status.setDiagnostics("Service " + service.getName() +
- " version " + service.getVersion() + " saved.");
- status.setState(ServiceState.ACCEPTED);
- return formatResponse(Status.ACCEPTED, status);
- }
-
- private Response processComponentsUpgrade(UserGroupInformation ugi,
- String serviceName, Set<String> compNames) throws YarnException,
- IOException, InterruptedException {
- Service service = getServiceFromClient(ugi, serviceName);
- if (service.getState() != ServiceState.UPGRADING) {
- throw new YarnException(
- String.format("The upgrade of service %s has not been initiated.",
- service.getName()));
- }
- List<Container> containersToUpgrade = ServiceApiUtil
- .validateAndResolveCompsUpgrade(service, compNames);
- Integer result = invokeContainersUpgrade(ugi, service, containersToUpgrade);
- if (result == EXIT_SUCCESS) {
- ServiceStatus status = new ServiceStatus();
- status.setDiagnostics(
- "Upgrading components " + Joiner.on(',').join(compNames) + ".");
- return formatResponse(Response.Status.ACCEPTED, status);
- }
- // If result is not a success, consider it a no-op
- return Response.status(Response.Status.NO_CONTENT).build();
- }
-
- private Response processContainersUpgrade(UserGroupInformation ugi,
- Service service, List<Container> containers) throws YarnException,
- IOException, InterruptedException {
-
- if (service.getState() != ServiceState.UPGRADING) {
- throw new YarnException(
- String.format("The upgrade of service %s has not been initiated.",
- service.getName()));
- }
- ServiceApiUtil.validateInstancesUpgrade(containers);
- Integer result = invokeContainersUpgrade(ugi, service, containers);
- if (result == EXIT_SUCCESS) {
- ServiceStatus status = new ServiceStatus();
- status.setDiagnostics(
- "Upgrading component instances " + containers.stream()
- .map(Container::getId).collect(Collectors.joining(",")) + ".");
- return formatResponse(Response.Status.ACCEPTED, status);
- }
- // If result is not a success, consider it a no-op
- return Response.status(Response.Status.NO_CONTENT).build();
- }
-
- private int invokeContainersUpgrade(UserGroupInformation ugi,
- Service service, List<Container> containers) throws IOException,
- InterruptedException {
- return ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
- int result1;
- ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result1 = sc.actionUpgrade(service, containers);
- sc.close();
- return result1;
- });
- }
-
- private Service getServiceFromClient(UserGroupInformation ugi,
- String serviceName) throws IOException, InterruptedException {
-
- return ugi.doAs((PrivilegedExceptionAction<Service>) () -> {
- ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Service app1 = sc.getStatus(serviceName);
- sc.close();
- return app1;
- });
- }
-
- /**
- * Used by negative test case.
- *
- * @param mockServerClient - A mocked version of ServiceClient
- */
- public void setServiceClient(ServiceClient mockServerClient) {
- serviceClientUnitTest = mockServerClient;
- unitTest = true;
- }
-
- private ServiceClient getServiceClient() {
- if (unitTest) {
- return serviceClientUnitTest;
- } else {
- return new ServiceClient();
- }
- }
-
- /**
- * Configure impersonation callback.
- *
- * @param request - web request
- * @return - configured UGI class for proxy callback
- * @throws IOException - if user is not login.
- */
- private UserGroupInformation getProxyUser(HttpServletRequest request)
- throws AccessControlException {
- UserGroupInformation proxyUser;
- UserGroupInformation ugi;
- String remoteUser = request.getRemoteUser();
- try {
- if (UserGroupInformation.isSecurityEnabled()) {
- proxyUser = UserGroupInformation.getLoginUser();
- ugi = UserGroupInformation.createProxyUser(remoteUser, proxyUser);
- } else {
- ugi = UserGroupInformation.createRemoteUser(remoteUser);
- }
- return ugi;
- } catch (IOException e) {
- throw new AccessControlException(e.getCause());
- }
- }
-
- /**
- * Format HTTP response.
- *
- * @param status - HTTP Code
- * @param message - Diagnostic message
- * @return - HTTP response
- */
- private Response formatResponse(Status status, String message) {
- ServiceStatus entity = new ServiceStatus();
- entity.setDiagnostics(message);
- return formatResponse(status, entity);
- }
-
- /**
- * Format HTTP response.
- *
- * @param status - HTTP Code
- * @param entity - ServiceStatus object
- * @return - HTTP response
- */
- private Response formatResponse(Status status, ServiceStatus entity) {
- return Response.status(status).entity(entity).build();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/6] hadoop git commit: YARN-7530. Refactored YARN service API
project location. Contributed by Chandni Singh
Posted by ey...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
new file mode 100644
index 0000000..f9cfa92
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.service.SystemServiceManager;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
+
+/**
+ * SystemServiceManager implementation.
+ * Scan for configure system service path.
+ *
+ * The service path structure is as follows:
+ * SYSTEM_SERVICE_DIR_PATH
+ * |---- sync
+ * | |--- user1
+ * | | |---- service1.yarnfile
+ * | | |---- service2.yarnfile
+ * | |--- user2
+ * | | |---- service1.yarnfile
+ * | | ....
+ * | |
+ * |---- async
+ * | |--- user3
+ * | | |---- service1.yarnfile
+ * | | |---- service2.yarnfile
+ * | |--- user4
+ * | | |---- service1.yarnfile
+ * | | ....
+ * | |
+ *
+ * sync: These services are launched at the time of service start synchronously.
+ * It is a blocking service start.
+ * async: These services are launched in separate thread without any delay after
+ * service start. Non-blocking service start.
+ */
+public class SystemServiceManagerImpl extends AbstractService
+ implements SystemServiceManager {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SystemServiceManagerImpl.class);
+
+ private static final String YARN_FILE_SUFFIX = ".yarnfile";
+ private static final String SYNC = "sync";
+ private static final String ASYNC = "async";
+
+ private FileSystem fs;
+ private Path systemServiceDir;
+ private AtomicBoolean stopExecutors = new AtomicBoolean(false);
+ private Map<String, Set<Service>> syncUserServices = new HashMap<>();
+ private Map<String, Set<Service>> asyncUserServices = new HashMap<>();
+ private UserGroupInformation loginUGI;
+ private Thread serviceLaucher;
+
+ @VisibleForTesting
+ private int badFileNameExtensionSkipCounter;
+ @VisibleForTesting
+ private Map<String, Integer> ignoredUserServices =
+ new HashMap<>();
+ @VisibleForTesting
+ private int badDirSkipCounter;
+
+ public SystemServiceManagerImpl() {
+ super(SystemServiceManagerImpl.class.getName());
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ String dirPath =
+ conf.get(YarnServiceConf.YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY);
+ if (dirPath != null) {
+ systemServiceDir = new Path(dirPath);
+ LOG.info("System Service Directory is configured to {}",
+ systemServiceDir);
+ fs = systemServiceDir.getFileSystem(conf);
+ this.loginUGI = UserGroupInformation.isSecurityEnabled() ?
+ UserGroupInformation.getLoginUser() :
+ UserGroupInformation.getCurrentUser();
+ LOG.info("UserGroupInformation initialized to {}", loginUGI);
+ }
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ scanForUserServices();
+ launchUserService(syncUserServices);
+ // Create a thread and submit services in background otherwise it
+ // block RM switch time.
+ serviceLaucher = new Thread(createRunnable());
+ serviceLaucher.setName("System service launcher");
+ serviceLaucher.start();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ LOG.info("Stopping {}", getName());
+ stopExecutors.set(true);
+
+ if (serviceLaucher != null) {
+ serviceLaucher.interrupt();
+ try {
+ serviceLaucher.join();
+ } catch (InterruptedException ie) {
+ LOG.warn("Interrupted Exception while stopping", ie);
+ }
+ }
+ }
+
+ private Runnable createRunnable() {
+ return new Runnable() {
+ @Override
+ public void run() {
+ launchUserService(asyncUserServices);
+ }
+ };
+ }
+
+ void launchUserService(Map<String, Set<Service>> userServices) {
+ for (Map.Entry<String, Set<Service>> entry : userServices.entrySet()) {
+ String user = entry.getKey();
+ Set<Service> services = entry.getValue();
+ if (services.isEmpty()) {
+ continue;
+ }
+ ServiceClient serviceClient = null;
+ try {
+ UserGroupInformation userUgi = getProxyUser(user);
+ serviceClient = createServiceClient(userUgi);
+ for (Service service : services) {
+ LOG.info("POST: createService = {} user = {}", service, userUgi);
+ try {
+ launchServices(userUgi, serviceClient, service);
+ } catch (IOException | UndeclaredThrowableException e) {
+ if (e.getCause() != null) {
+ LOG.warn(e.getCause().getMessage());
+ } else {
+ String message =
+ "Failed to create service " + service.getName() + " : ";
+ LOG.error(message, e);
+ }
+ }
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("System service launcher thread interrupted", e);
+ break;
+ } catch (Exception e) {
+ LOG.error("Error while submitting services for user " + user, e);
+ } finally {
+ if (serviceClient != null) {
+ try {
+ serviceClient.close();
+ } catch (IOException e) {
+ LOG.warn("Error while closing serviceClient for user {}", user);
+ }
+ }
+ }
+ }
+ }
+
+ private ServiceClient createServiceClient(UserGroupInformation userUgi)
+ throws IOException, InterruptedException {
+ ServiceClient serviceClient =
+ userUgi.doAs(new PrivilegedExceptionAction<ServiceClient>() {
+ @Override public ServiceClient run()
+ throws IOException, YarnException {
+ ServiceClient sc = getServiceClient();
+ sc.init(getConfig());
+ sc.start();
+ return sc;
+ }
+ });
+ return serviceClient;
+ }
+
+ private void launchServices(UserGroupInformation userUgi,
+ ServiceClient serviceClient, Service service)
+ throws IOException, InterruptedException {
+ if (service.getState() == ServiceState.STOPPED) {
+ userUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override public Void run() throws IOException, YarnException {
+ serviceClient.actionBuild(service);
+ return null;
+ }
+ });
+ LOG.info("Service {} version {} saved.", service.getName(),
+ service.getVersion());
+ } else {
+ ApplicationId applicationId =
+ userUgi.doAs(new PrivilegedExceptionAction<ApplicationId>() {
+ @Override public ApplicationId run()
+ throws IOException, YarnException {
+ ApplicationId applicationId = serviceClient.actionCreate(service);
+ return applicationId;
+ }
+ });
+ LOG.info("Service {} submitted with Application ID: {}",
+ service.getName(), applicationId);
+ }
+ }
+
+ ServiceClient getServiceClient() {
+ return new ServiceClient();
+ }
+
+ private UserGroupInformation getProxyUser(String user) {
+ UserGroupInformation ugi;
+ if (UserGroupInformation.isSecurityEnabled()) {
+ ugi = UserGroupInformation.createProxyUser(user, loginUGI);
+ } else {
+ ugi = UserGroupInformation.createRemoteUser(user);
+ }
+ return ugi;
+ }
+
+ // scan for both launch service types i.e sync and async
+ void scanForUserServices() throws IOException {
+ if (systemServiceDir == null) {
+ return;
+ }
+ try {
+ LOG.info("Scan for launch type on {}", systemServiceDir);
+ RemoteIterator<FileStatus> iterLaunchType = list(systemServiceDir);
+ while (iterLaunchType.hasNext()) {
+ FileStatus launchType = iterLaunchType.next();
+ if (!launchType.isDirectory()) {
+ LOG.debug("Scanner skips for unknown file {}", launchType.getPath());
+ continue;
+ }
+ if (launchType.getPath().getName().equals(SYNC)) {
+ scanForUserServiceDefinition(launchType.getPath(), syncUserServices);
+ } else if (launchType.getPath().getName().equals(ASYNC)) {
+ scanForUserServiceDefinition(launchType.getPath(), asyncUserServices);
+ } else {
+ badDirSkipCounter++;
+ LOG.debug("Scanner skips for unknown dir {}.", launchType.getPath());
+ }
+ }
+ } catch (FileNotFoundException e) {
+ LOG.warn("System service directory {} doesn't not exist.",
+ systemServiceDir);
+ }
+ }
+
+ // Files are under systemServiceDir/<users>. Scan for 2 levels
+ // 1st level for users
+ // 2nd level for service definitions under user
+ private void scanForUserServiceDefinition(Path userDirPath,
+ Map<String, Set<Service>> userServices) throws IOException {
+ LOG.info("Scan for users on {}", userDirPath);
+ RemoteIterator<FileStatus> iterUsers = list(userDirPath);
+ while (iterUsers.hasNext()) {
+ FileStatus userDir = iterUsers.next();
+ // if 1st level is not user directory then skip it.
+ if (!userDir.isDirectory()) {
+ LOG.info(
+ "Service definition {} doesn't belong to any user. Ignoring.. ",
+ userDir.getPath().getName());
+ continue;
+ }
+ String userName = userDir.getPath().getName();
+ LOG.info("Scanning service definitions for user {}.", userName);
+
+ //2nd level scan
+ RemoteIterator<FileStatus> iterServices = list(userDir.getPath());
+ while (iterServices.hasNext()) {
+ FileStatus serviceCache = iterServices.next();
+ String filename = serviceCache.getPath().getName();
+ if (!serviceCache.isFile()) {
+ LOG.info("Scanner skips for unknown dir {}", filename);
+ continue;
+ }
+ if (!filename.endsWith(YARN_FILE_SUFFIX)) {
+ LOG.info("Scanner skips for unknown file extension, filename = {}",
+ filename);
+ badFileNameExtensionSkipCounter++;
+ continue;
+ }
+ Service service = getServiceDefinition(serviceCache.getPath());
+ if (service != null) {
+ Set<Service> services = userServices.get(userName);
+ if (services == null) {
+ services = new HashSet<>();
+ userServices.put(userName, services);
+ }
+ if (!services.add(service)) {
+ int count = ignoredUserServices.containsKey(userName) ?
+ ignoredUserServices.get(userName) : 0;
+ ignoredUserServices.put(userName, count + 1);
+ LOG.warn(
+ "Ignoring service {} for the user {} as it is already present,"
+ + " filename = {}", service.getName(), userName, filename);
+ } else {
+ LOG.info("Added service {} for the user {}, filename = {}",
+ service.getName(), userName, filename);
+ }
+ }
+ }
+ }
+ }
+
+ private Service getServiceDefinition(Path filePath) {
+ Service service = null;
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading service definition from FS: " + filePath);
+ }
+ service = jsonSerDeser.load(fs, filePath);
+ } catch (IOException e) {
+ LOG.info("Error while loading service definition from FS: {}", e);
+ }
+ return service;
+ }
+
+ private RemoteIterator<FileStatus> list(Path path) throws IOException {
+ return new StoppableRemoteIterator(fs.listStatusIterator(path));
+ }
+
+ @VisibleForTesting Map<String, Integer> getIgnoredUserServices() {
+ return ignoredUserServices;
+ }
+
+ private class StoppableRemoteIterator implements RemoteIterator<FileStatus> {
+ private final RemoteIterator<FileStatus> remote;
+
+ StoppableRemoteIterator(RemoteIterator<FileStatus> remote) {
+ this.remote = remote;
+ }
+
+ @Override public boolean hasNext() throws IOException {
+ return !stopExecutors.get() && remote.hasNext();
+ }
+
+ @Override public FileStatus next() throws IOException {
+ return remote.next();
+ }
+ }
+
+ @VisibleForTesting
+ Map<String, Set<Service>> getSyncUserServices() {
+ return syncUserServices;
+ }
+
+ @VisibleForTesting
+ int getBadFileNameExtensionSkipCounter() {
+ return badFileNameExtensionSkipCounter;
+ }
+
+ @VisibleForTesting
+ int getBadDirSkipCounter() {
+ return badDirSkipCounter;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java
new file mode 100644
index 0000000..cf5ce11
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.service.client contains classes
+ * for YARN Services Client API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
new file mode 100644
index 0000000..46c9abe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -0,0 +1,818 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.webapp;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ComponentState;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.conf.RestApiConstants;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED;
+import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
+
+/**
+ * The rest API endpoints for users to manage services on YARN.
+ */
+@Singleton
+@Path(CONTEXT_ROOT)
+public class ApiServer {
+
+ public ApiServer() {
+ super();
+ }
+
+ @Inject
+ public ApiServer(Configuration conf) {
+ super();
+ }
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApiServer.class);
+ private static Configuration YARN_CONFIG = new YarnConfiguration();
+ private ServiceClient serviceClientUnitTest;
+ private boolean unitTest = false;
+
+ static {
+ init();
+ }
+
+ // initialize all the common resources - order is important
+ private static void init() {
+ }
+
+ @GET
+ @Path(VERSION)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
+ public Response getVersion() {
+ String version = VersionInfo.getBuildVersion();
+ LOG.info(version);
+ return Response.ok("{ \"hadoop_version\": \"" + version + "\"}").build();
+ }
+
+ @POST
+ @Path(SERVICE_ROOT_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
+ public Response createService(@Context HttpServletRequest request,
+ Service service) {
+ ServiceStatus serviceStatus = new ServiceStatus();
+ try {
+ UserGroupInformation ugi = getProxyUser(request);
+ LOG.info("POST: createService = {} user = {}", service, ugi);
+ if(service.getState()==ServiceState.STOPPED) {
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws YarnException, IOException {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ sc.actionBuild(service);
+ sc.close();
+ return null;
+ }
+ });
+ serviceStatus.setDiagnostics("Service " + service.getName() +
+ " version " + service.getVersion() + " saved.");
+ } else {
+ ApplicationId applicationId = ugi
+ .doAs(new PrivilegedExceptionAction<ApplicationId>() {
+ @Override
+ public ApplicationId run() throws IOException, YarnException {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ ApplicationId applicationId = sc.actionCreate(service);
+ sc.close();
+ return applicationId;
+ }
+ });
+ serviceStatus.setDiagnostics("Application ID: " + applicationId);
+ }
+ serviceStatus.setState(ACCEPTED);
+ serviceStatus.setUri(
+ CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
+ .getName());
+ return formatResponse(Status.ACCEPTED, serviceStatus);
+ } catch (AccessControlException e) {
+ serviceStatus.setDiagnostics(e.getMessage());
+ return formatResponse(Status.FORBIDDEN, e.getCause().getMessage());
+ } catch (IllegalArgumentException e) {
+ return formatResponse(Status.BAD_REQUEST, e.getMessage());
+ } catch (IOException | InterruptedException e) {
+ String message = "Failed to create service " + service.getName()
+ + ": {}";
+ LOG.error(message, e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ String message = "Failed to create service " + service.getName()
+ + ": {}";
+ LOG.error(message, e);
+ if (e.getCause().getMessage().contains("already exists")) {
+ message = "Service name " + service.getName() + " is already taken.";
+ } else {
+ message = e.getCause().getMessage();
+ }
+ return formatResponse(Status.INTERNAL_SERVER_ERROR,
+ message);
+ }
+ }
+
+ @GET
+ @Path(SERVICE_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
+ public Response getService(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String appName) {
+ ServiceStatus serviceStatus = new ServiceStatus();
+ try {
+ if (appName == null) {
+ throw new IllegalArgumentException("Service name cannot be null.");
+ }
+ UserGroupInformation ugi = getProxyUser(request);
+ LOG.info("GET: getService for appName = {} user = {}", appName, ugi);
+ Service app = getServiceFromClient(ugi, appName);
+ return Response.ok(app).build();
+ } catch (AccessControlException e) {
+ return formatResponse(Status.FORBIDDEN, e.getMessage());
+ } catch (IllegalArgumentException e) {
+ serviceStatus.setDiagnostics(e.getMessage());
+ serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
+ return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+ .build();
+ } catch (FileNotFoundException e) {
+ serviceStatus.setDiagnostics("Service " + appName + " not found");
+ serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
+ return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+ .build();
+ } catch (IOException | InterruptedException e) {
+ LOG.error("Get service failed: {}", e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ LOG.error("Get service failed: {}", e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR,
+ e.getCause().getMessage());
+ }
+ }
+
+ @DELETE
+ @Path(SERVICE_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
+ public Response deleteService(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String appName) {
+ try {
+ if (appName == null) {
+ throw new IllegalArgumentException("Service name can not be null.");
+ }
+ UserGroupInformation ugi = getProxyUser(request);
+ LOG.info("DELETE: deleteService for appName = {} user = {}",
+ appName, ugi);
+ return stopService(appName, true, ugi);
+ } catch (AccessControlException e) {
+ return formatResponse(Status.FORBIDDEN, e.getMessage());
+ } catch (IllegalArgumentException e) {
+ return formatResponse(Status.BAD_REQUEST, e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ LOG.error("Fail to stop service: {}", e);
+ return formatResponse(Status.BAD_REQUEST,
+ e.getCause().getMessage());
+ } catch (YarnException | FileNotFoundException e) {
+ return formatResponse(Status.NOT_FOUND, e.getMessage());
+ } catch (Exception e) {
+ LOG.error("Fail to stop service: {}", e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+ }
+ }
+
+ private Response stopService(String appName, boolean destroy,
+ final UserGroupInformation ugi) throws Exception {
+ int result = ugi.doAs(new PrivilegedExceptionAction<Integer>() {
+ @Override
+ public Integer run() throws Exception {
+ int result = 0;
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Exception stopException = null;
+ try {
+ result = sc.actionStop(appName, destroy);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully stopped service {}", appName);
+ }
+ } catch (Exception e) {
+ LOG.info("Got exception stopping service", e);
+ stopException = e;
+ }
+ if (destroy) {
+ result = sc.actionDestroy(appName);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully deleted service {}", appName);
+ }
+ } else {
+ if (stopException != null) {
+ throw stopException;
+ }
+ }
+ sc.close();
+ return result;
+ }
+ });
+ ServiceStatus serviceStatus = new ServiceStatus();
+ if (destroy) {
+ if (result == EXIT_SUCCESS) {
+ serviceStatus.setDiagnostics("Successfully destroyed service " +
+ appName);
+ } else {
+ if (result == EXIT_NOT_FOUND) {
+ serviceStatus
+ .setDiagnostics("Service " + appName + " doesn't exist");
+ return formatResponse(Status.BAD_REQUEST, serviceStatus);
+ } else {
+ serviceStatus
+ .setDiagnostics("Service " + appName + " error cleaning up " +
+ "registry");
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, serviceStatus);
+ }
+ }
+ } else {
+ if (result == EXIT_COMMAND_ARGUMENT_ERROR) {
+ serviceStatus
+ .setDiagnostics("Service " + appName + " is already stopped");
+ return formatResponse(Status.BAD_REQUEST, serviceStatus);
+ } else {
+ serviceStatus.setDiagnostics("Successfully stopped service " + appName);
+ }
+ }
+ return formatResponse(Status.OK, serviceStatus);
+ }
+
+ @PUT
+ @Path(COMPONENTS_PATH)
+ @Consumes({MediaType.APPLICATION_JSON})
+ @Produces({RestApiConstants.MEDIA_TYPE_JSON_UTF8, MediaType.TEXT_PLAIN})
+ public Response updateComponents(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String serviceName,
+ List<Component> requestComponents) {
+
+ try {
+ if (requestComponents == null || requestComponents.isEmpty()) {
+ throw new YarnException("No components provided.");
+ }
+ UserGroupInformation ugi = getProxyUser(request);
+ Set<String> compNamesToUpgrade = new HashSet<>();
+ requestComponents.forEach(reqComp -> {
+ if (reqComp.getState() != null &&
+ reqComp.getState().equals(ComponentState.UPGRADING)) {
+ compNamesToUpgrade.add(reqComp.getName());
+ }
+ });
+ LOG.info("PUT: upgrade components {} for service {} " +
+ "user = {}", compNamesToUpgrade, serviceName, ugi);
+ return processComponentsUpgrade(ugi, serviceName, compNamesToUpgrade);
+ } catch (AccessControlException e) {
+ return formatResponse(Response.Status.FORBIDDEN, e.getMessage());
+ } catch (YarnException e) {
+ return formatResponse(Response.Status.BAD_REQUEST, e.getMessage());
+ } catch (IOException | InterruptedException e) {
+ return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
+ e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
+ e.getCause().getMessage());
+ }
+ }
+
+ @PUT
+ @Path(COMPONENT_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8",
+ MediaType.TEXT_PLAIN })
+ public Response updateComponent(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String appName,
+ @PathParam(COMPONENT_NAME) String componentName, Component component) {
+
+ try {
+ if (component == null) {
+ throw new YarnException("No component data provided");
+ }
+ if (component.getName() != null
+ && !component.getName().equals(componentName)) {
+ String msg = "Component name in the request object ("
+ + component.getName() + ") does not match that in the URI path ("
+ + componentName + ")";
+ throw new YarnException(msg);
+ }
+ UserGroupInformation ugi = getProxyUser(request);
+ if (component.getState() != null &&
+ component.getState().equals(ComponentState.UPGRADING)) {
+ LOG.info("PUT: upgrade component {} for service {} " +
+ "user = {}", component.getName(), appName, ugi);
+ return processComponentsUpgrade(ugi, appName,
+ Sets.newHashSet(componentName));
+ }
+
+ if (component.getNumberOfContainers() == null) {
+ throw new YarnException("No container count provided");
+ }
+ if (component.getNumberOfContainers() < 0) {
+ String message = "Invalid number of containers specified "
+ + component.getNumberOfContainers();
+ throw new YarnException(message);
+ }
+ Map<String, Long> original = ugi
+ .doAs(new PrivilegedExceptionAction<Map<String, Long>>() {
+ @Override
+ public Map<String, Long> run() throws YarnException, IOException {
+ ServiceClient sc = new ServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Map<String, Long> original = sc.flexByRestService(appName,
+ Collections.singletonMap(componentName,
+ component.getNumberOfContainers()));
+ sc.close();
+ return original;
+ }
+ });
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics(
+ "Updating component (" + componentName + ") size from " + original
+ .get(componentName) + " to " + component.getNumberOfContainers());
+ return formatResponse(Status.OK, status);
+ } catch (AccessControlException e) {
+ return formatResponse(Status.FORBIDDEN, e.getMessage());
+ } catch (YarnException e) {
+ return formatResponse(Status.BAD_REQUEST, e.getMessage());
+ } catch (IOException | InterruptedException e) {
+ return formatResponse(Status.INTERNAL_SERVER_ERROR,
+ e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ return formatResponse(Status.INTERNAL_SERVER_ERROR,
+ e.getCause().getMessage());
+ }
+ }
+
+ @PUT
+ @Path(SERVICE_PATH)
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" })
+ public Response updateService(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String appName,
+ Service updateServiceData) {
+ try {
+ UserGroupInformation ugi = getProxyUser(request);
+ LOG.info("PUT: updateService for app = {} with data = {} user = {}",
+ appName, updateServiceData, ugi);
+ // Ignore the app name provided in updateServiceData and always use
+ // appName path param
+ updateServiceData.setName(appName);
+
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.FLEX) {
+ return flexService(updateServiceData, ugi);
+ }
+ // For STOP the app should be running. If already stopped then this
+ // operation will be a no-op. For START it should be in stopped state.
+ // If already running then this operation will be a no-op.
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STOPPED) {
+ return stopService(appName, false, ugi);
+ }
+
+ // If a START is requested
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STARTED) {
+ return startService(appName, ugi);
+ }
+
+ // If an UPGRADE is requested
+ if (updateServiceData.getState() != null && (
+ updateServiceData.getState() == ServiceState.UPGRADING ||
+ updateServiceData.getState() ==
+ ServiceState.UPGRADING_AUTO_FINALIZE)) {
+ return upgradeService(updateServiceData, ugi);
+ }
+
+ // If new lifetime value specified then update it
+ if (updateServiceData.getLifetime() != null
+ && updateServiceData.getLifetime() > 0) {
+ return updateLifetime(appName, updateServiceData, ugi);
+ }
+ } catch (UndeclaredThrowableException e) {
+ return formatResponse(Status.BAD_REQUEST,
+ e.getCause().getMessage());
+ } catch (AccessControlException e) {
+ return formatResponse(Status.FORBIDDEN, e.getMessage());
+ } catch (FileNotFoundException e) {
+ String message = "Application is not found app: " + appName;
+ LOG.error(message, e);
+ return formatResponse(Status.NOT_FOUND, e.getMessage());
+ } catch (YarnException e) {
+ String message = "Service is not found in hdfs: " + appName;
+ LOG.error(message, e);
+ return formatResponse(Status.NOT_FOUND, e.getMessage());
+ } catch (Exception e) {
+ String message = "Error while performing operation for app: " + appName;
+ LOG.error(message, e);
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+ }
+
+ // If nothing happens consider it a no-op
+ return Response.status(Status.NO_CONTENT).build();
+ }
+
+ @PUT
+ @Path(COMP_INSTANCE_LONG_PATH)
+ @Consumes({MediaType.APPLICATION_JSON})
+ @Produces({RestApiConstants.MEDIA_TYPE_JSON_UTF8, MediaType.TEXT_PLAIN})
+ public Response updateComponentInstance(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String serviceName,
+ @PathParam(COMPONENT_NAME) String componentName,
+ @PathParam(COMP_INSTANCE_NAME) String compInstanceName,
+ Container reqContainer) {
+
+ try {
+ UserGroupInformation ugi = getProxyUser(request);
+ LOG.info("PUT: update component instance {} for component = {}" +
+ " service = {} user = {}", compInstanceName, componentName,
+ serviceName, ugi);
+ if (reqContainer == null) {
+ throw new YarnException("No container data provided.");
+ }
+ Service service = getServiceFromClient(ugi, serviceName);
+ Component component = service.getComponent(componentName);
+ if (component == null) {
+ throw new YarnException(String.format(
+ "The component name in the URI path (%s) is invalid.",
+ componentName));
+ }
+
+ Container liveContainer = component.getComponentInstance(
+ compInstanceName);
+ if (liveContainer == null) {
+ throw new YarnException(String.format(
+ "The component (%s) does not have a component instance (%s).",
+ componentName, compInstanceName));
+ }
+
+ if (reqContainer.getState() != null
+ && reqContainer.getState().equals(ContainerState.UPGRADING)) {
+ return processContainersUpgrade(ugi, service,
+ Lists.newArrayList(liveContainer));
+ }
+ } catch (AccessControlException e) {
+ return formatResponse(Response.Status.FORBIDDEN, e.getMessage());
+ } catch (YarnException e) {
+ return formatResponse(Response.Status.BAD_REQUEST, e.getMessage());
+ } catch (IOException | InterruptedException e) {
+ return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
+ e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
+ e.getCause().getMessage());
+ }
+ return Response.status(Status.NO_CONTENT).build();
+ }
+
+ @PUT
+ @Path(COMP_INSTANCES_PATH)
+ @Consumes({MediaType.APPLICATION_JSON})
+ @Produces({RestApiConstants.MEDIA_TYPE_JSON_UTF8, MediaType.TEXT_PLAIN})
+ public Response updateComponentInstances(@Context HttpServletRequest request,
+ @PathParam(SERVICE_NAME) String serviceName,
+ List<Container> requestContainers) {
+
+ try {
+ if (requestContainers == null || requestContainers.isEmpty()) {
+ throw new YarnException("No containers provided.");
+ }
+ UserGroupInformation ugi = getProxyUser(request);
+ List<String> toUpgrade = new ArrayList<>();
+ for (Container reqContainer : requestContainers) {
+ if (reqContainer.getState() != null &&
+ reqContainer.getState().equals(ContainerState.UPGRADING)) {
+ toUpgrade.add(reqContainer.getComponentInstanceName());
+ }
+ }
+
+ if (!toUpgrade.isEmpty()) {
+ Service service = getServiceFromClient(ugi, serviceName);
+ LOG.info("PUT: upgrade component instances {} for service = {} " +
+ "user = {}", toUpgrade, serviceName, ugi);
+ List<Container> liveContainers = ServiceApiUtil
+ .getLiveContainers(service, toUpgrade);
+
+ return processContainersUpgrade(ugi, service, liveContainers);
+ }
+ } catch (AccessControlException e) {
+ return formatResponse(Response.Status.FORBIDDEN, e.getMessage());
+ } catch (YarnException e) {
+ return formatResponse(Response.Status.BAD_REQUEST, e.getMessage());
+ } catch (IOException | InterruptedException e) {
+ return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
+ e.getMessage());
+ } catch (UndeclaredThrowableException e) {
+ return formatResponse(Response.Status.INTERNAL_SERVER_ERROR,
+ e.getCause().getMessage());
+ }
+ return Response.status(Status.NO_CONTENT).build();
+ }
+
+ private Response flexService(Service service, UserGroupInformation ugi)
+ throws IOException, InterruptedException {
+ String appName = service.getName();
+ Response response = Response.status(Status.BAD_REQUEST).build();
+ Map<String, String> componentCountStrings = new HashMap<String, String>();
+ for (Component c : service.getComponents()) {
+ componentCountStrings.put(c.getName(),
+ c.getNumberOfContainers().toString());
+ }
+ Integer result = ugi.doAs(new PrivilegedExceptionAction<Integer>() {
+
+ @Override
+ public Integer run() throws YarnException, IOException {
+ int result = 0;
+ ServiceClient sc = new ServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc
+ .actionFlex(appName, componentCountStrings);
+ sc.close();
+ return Integer.valueOf(result);
+ }
+ });
+ if (result == EXIT_SUCCESS) {
+ String message = "Service " + appName + " is successfully flexed.";
+ LOG.info(message);
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics(message);
+ status.setState(ServiceState.ACCEPTED);
+ response = formatResponse(Status.ACCEPTED, status);
+ }
+ return response;
+ }
+
+ private Response updateLifetime(String appName, Service updateAppData,
+ final UserGroupInformation ugi) throws IOException,
+ InterruptedException {
+ String newLifeTime = ugi.doAs(new PrivilegedExceptionAction<String>() {
+ @Override
+ public String run() throws YarnException, IOException {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ String newLifeTime = sc.updateLifetime(appName,
+ updateAppData.getLifetime());
+ sc.close();
+ return newLifeTime;
+ }
+ });
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics(
+ "Service (" + appName + ")'s lifeTime is updated to " + newLifeTime
+ + ", " + updateAppData.getLifetime() + " seconds remaining");
+ return formatResponse(Status.OK, status);
+ }
+
+ private Response startService(String appName,
+ final UserGroupInformation ugi) throws IOException,
+ InterruptedException {
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws YarnException, IOException {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ sc.actionStart(appName);
+ sc.close();
+ return null;
+ }
+ });
+ LOG.info("Successfully started service " + appName);
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics("Service " + appName + " is successfully started.");
+ status.setState(ServiceState.ACCEPTED);
+ return formatResponse(Status.OK, status);
+ }
+
+ private Response upgradeService(Service service,
+ final UserGroupInformation ugi) throws IOException, InterruptedException {
+ ServiceStatus status = new ServiceStatus();
+ ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ sc.initiateUpgrade(service);
+ sc.close();
+ return null;
+ });
+ LOG.info("Service {} version {} upgrade initialized", service.getName(),
+ service.getVersion());
+ status.setDiagnostics("Service " + service.getName() +
+ " version " + service.getVersion() + " saved.");
+ status.setState(ServiceState.ACCEPTED);
+ return formatResponse(Status.ACCEPTED, status);
+ }
+
+ private Response processComponentsUpgrade(UserGroupInformation ugi,
+ String serviceName, Set<String> compNames) throws YarnException,
+ IOException, InterruptedException {
+ Service service = getServiceFromClient(ugi, serviceName);
+ if (service.getState() != ServiceState.UPGRADING) {
+ throw new YarnException(
+ String.format("The upgrade of service %s has not been initiated.",
+ service.getName()));
+ }
+ List<Container> containersToUpgrade = ServiceApiUtil
+ .validateAndResolveCompsUpgrade(service, compNames);
+ Integer result = invokeContainersUpgrade(ugi, service, containersToUpgrade);
+ if (result == EXIT_SUCCESS) {
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics(
+ "Upgrading components " + Joiner.on(',').join(compNames) + ".");
+ return formatResponse(Response.Status.ACCEPTED, status);
+ }
+ // If result is not a success, consider it a no-op
+ return Response.status(Response.Status.NO_CONTENT).build();
+ }
+
+ private Response processContainersUpgrade(UserGroupInformation ugi,
+ Service service, List<Container> containers) throws YarnException,
+ IOException, InterruptedException {
+
+ if (service.getState() != ServiceState.UPGRADING) {
+ throw new YarnException(
+ String.format("The upgrade of service %s has not been initiated.",
+ service.getName()));
+ }
+ ServiceApiUtil.validateInstancesUpgrade(containers);
+ Integer result = invokeContainersUpgrade(ugi, service, containers);
+ if (result == EXIT_SUCCESS) {
+ ServiceStatus status = new ServiceStatus();
+ status.setDiagnostics(
+ "Upgrading component instances " + containers.stream()
+ .map(Container::getId).collect(Collectors.joining(",")) + ".");
+ return formatResponse(Response.Status.ACCEPTED, status);
+ }
+ // If result is not a success, consider it a no-op
+ return Response.status(Response.Status.NO_CONTENT).build();
+ }
+
+ private int invokeContainersUpgrade(UserGroupInformation ugi,
+ Service service, List<Container> containers) throws IOException,
+ InterruptedException {
+ return ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
+ int result1;
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result1 = sc.actionUpgrade(service, containers);
+ sc.close();
+ return result1;
+ });
+ }
+
+ private Service getServiceFromClient(UserGroupInformation ugi,
+ String serviceName) throws IOException, InterruptedException {
+
+ return ugi.doAs((PrivilegedExceptionAction<Service>) () -> {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Service app1 = sc.getStatus(serviceName);
+ sc.close();
+ return app1;
+ });
+ }
+
+ /**
+ * Used by negative test case.
+ *
+ * @param mockServerClient - A mocked version of ServiceClient
+ */
+ public void setServiceClient(ServiceClient mockServerClient) {
+ serviceClientUnitTest = mockServerClient;
+ unitTest = true;
+ }
+
+ private ServiceClient getServiceClient() {
+ if (unitTest) {
+ return serviceClientUnitTest;
+ } else {
+ return new ServiceClient();
+ }
+ }
+
+ /**
+ * Configure impersonation callback.
+ *
+ * @param request - web request
+ * @return - configured UGI class for proxy callback
+ * @throws IOException - if user is not login.
+ */
+ private UserGroupInformation getProxyUser(HttpServletRequest request)
+ throws AccessControlException {
+ UserGroupInformation proxyUser;
+ UserGroupInformation ugi;
+ String remoteUser = request.getRemoteUser();
+ try {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ proxyUser = UserGroupInformation.getLoginUser();
+ ugi = UserGroupInformation.createProxyUser(remoteUser, proxyUser);
+ } else {
+ ugi = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ return ugi;
+ } catch (IOException e) {
+ throw new AccessControlException(e.getCause());
+ }
+ }
+
+ /**
+ * Format HTTP response.
+ *
+ * @param status - HTTP Code
+ * @param message - Diagnostic message
+ * @return - HTTP response
+ */
+ private Response formatResponse(Status status, String message) {
+ ServiceStatus entity = new ServiceStatus();
+ entity.setDiagnostics(message);
+ return formatResponse(status, entity);
+ }
+
+ /**
+ * Format HTTP response.
+ *
+ * @param status - HTTP Code
+ * @param entity - ServiceStatus object
+ * @return - HTTP response
+ */
+ private Response formatResponse(Status status, ServiceStatus entity) {
+ return Response.status(status).entity(entity).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
new file mode 100644
index 0000000..f4acd94
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.webapp;
+
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.eclipse.jetty.webapp.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
+
+/**
+ * This class launches the web service using Hadoop HttpServer2 (which uses
+ * an embedded Jetty container). This is the entry point to your service.
+ * The Java command used to launch this app should call the main method.
+ */
+public class ApiServerWebApp extends AbstractService {
+ private static final Logger logger = LoggerFactory
+ .getLogger(ApiServerWebApp.class);
+ private static final String SEP = ";";
+
+ // REST API server for YARN native services
+ private HttpServer2 apiServer;
+ private InetSocketAddress bindAddress;
+
+ public static void main(String[] args) throws IOException {
+ ApiServerWebApp apiWebApp = new ApiServerWebApp();
+ try {
+ apiWebApp.init(new YarnConfiguration());
+ apiWebApp.serviceStart();
+ } catch (Exception e) {
+ logger.error("Got exception starting", e);
+ apiWebApp.close();
+ }
+ }
+
+ public ApiServerWebApp() {
+ super(ApiServerWebApp.class.getName());
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS,
+ DEFAULT_API_SERVER_ADDRESS, DEFAULT_API_SERVER_PORT);
+ logger.info("YARN API server running on " + bindAddress);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ doSecureLogin(getConfig());
+ }
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (apiServer != null) {
+ apiServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ private void doSecureLogin(org.apache.hadoop.conf.Configuration conf)
+ throws IOException {
+ SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB,
+ YarnConfiguration.RM_PRINCIPAL, bindAddress.getHostName());
+ addFilters(conf);
+ }
+
+ private void addFilters(org.apache.hadoop.conf.Configuration conf) {
+ // Always load pseudo authentication filter to parse "user.name" in an URL
+ // to identify a HTTP request's user.
+ boolean hasHadoopAuthFilterInitializer = false;
+ String filterInitializerConfKey = "hadoop.http.filter.initializers";
+ Class<?>[] initializersClasses =
+ conf.getClasses(filterInitializerConfKey);
+ List<String> targets = new ArrayList<String>();
+ if (initializersClasses != null) {
+ for (Class<?> initializer : initializersClasses) {
+ if (initializer.getName().equals(
+ AuthenticationFilterInitializer.class.getName())) {
+ hasHadoopAuthFilterInitializer = true;
+ break;
+ }
+ targets.add(initializer.getName());
+ }
+ }
+ if (!hasHadoopAuthFilterInitializer) {
+ targets.add(AuthenticationFilterInitializer.class.getName());
+ conf.set(filterInitializerConfKey, StringUtils.join(",", targets));
+ }
+ }
+
+ private void startWebApp() throws IOException {
+ URI uri = URI.create("http://" + NetUtils.getHostPortString(bindAddress));
+
+ apiServer = new HttpServer2.Builder()
+ .setName("api-server")
+ .setConf(getConfig())
+ .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
+ .setUsernameConfKey(RM_WEBAPP_SPNEGO_USER_NAME_KEY)
+ .setKeytabConfKey(RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+ .addEndpoint(uri).build();
+
+ String apiPackages =
+ ApiServer.class.getPackage().getName() + SEP
+ + GenericExceptionHandler.class.getPackage().getName() + SEP
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
+ apiServer.addJerseyResourcePackage(apiPackages, "/*");
+
+ try {
+ logger.info("Service starting up. Logging start...");
+ apiServer.start();
+ logger.info("Server status = {}", apiServer.toString());
+ for (Configuration conf : apiServer.getWebAppContext()
+ .getConfigurations()) {
+ logger.info("Configurations = {}", conf);
+ }
+ logger.info("Context Path = {}", Collections.singletonList(
+ apiServer.getWebAppContext().getContextPath()));
+ logger.info("ResourceBase = {}", Collections.singletonList(
+ apiServer.getWebAppContext().getResourceBase()));
+ logger.info("War = {}", Collections
+ .singletonList(apiServer.getWebAppContext().getWar()));
+ } catch (Exception ex) {
+ logger.error("Hadoop HttpServer2 App **failed**", ex);
+ throw ex;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java
new file mode 100644
index 0000000..1bdf05a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.service.webapp contains classes to be used
+ * for YARN Services API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.service.webapp;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
new file mode 100644
index 0000000..b7ad6c9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
@@ -0,0 +1,444 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+## Examples
+
+### Create a simple single-component service with most attribute values as defaults
+POST URL - http://localhost:8088/app/v1/services
+
+##### POST Request JSON
+```json
+{
+ "name": "hello-world",
+ "version": "1.0.0",
+ "description": "hello world example",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 2,
+ "artifact": {
+ "id": "nginx:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "./start_nginx.sh",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ }
+ }
+ ]
+}
+```
+
+##### GET Response JSON
+GET URL - http://localhost:8088/app/v1/services/hello-world
+
+Note, lifetime value of -1 means unlimited lifetime.
+
+```json
+{
+ "name": "hello-world",
+ "version": "1.0.0",
+ "description": "hello world example",
+ "id": "application_1503963985568_0002",
+ "lifetime": -1,
+ "state": "STABLE",
+ "components": [
+ {
+ "name": "hello",
+ "state": "STABLE",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": [],
+ "containers": [
+ {
+ "id": "container_e03_1503963985568_0002_01_000002",
+ "ip": "10.22.8.143",
+ "hostname": "ctr-e03-1503963985568-0002-01-000002.example.site",
+ "state": "READY",
+ "launch_time": 1504051512412,
+ "bare_host": "host100.cloud.com",
+ "component_instance_name": "hello-0"
+ },
+ {
+ "id": "container_e03_1503963985568_0002_01_000003",
+ "ip": "10.22.8.144",
+ "hostname": "ctr-e03-1503963985568-0002-01-000003.example.site",
+ "state": "READY",
+ "launch_time": 1504051536450,
+ "bare_host": "host100.cloud.com",
+ "component_instance_name": "hello-1"
+ }
+ ],
+ "launch_command": "./start_nginx.sh",
+ "number_of_containers": 1,
+ "run_privileged_container": false
+ }
+ ],
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": {}
+}
+
+```
+### Update to modify the lifetime of a service
+PUT URL - http://localhost:8088/app/v1/services/hello-world
+
+##### PUT Request JSON
+
+Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
+
+```json
+{
+ "lifetime": 3600
+}
+```
+### Stop a service
+PUT URL - http://localhost:8088/app/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STOPPED"
+}
+```
+
+### Start a service
+PUT URL - http://localhost:8088/app/v1/services/hello-world
+
+##### PUT Request JSON
+```json
+{
+ "state": "STARTED"
+}
+```
+
+### Update to flex up/down the number of containers (instances) of a component of a service
+PUT URL - http://localhost:8088/app/v1/services/hello-world/components/hello
+
+##### PUT Request JSON
+```json
+{
+ "number_of_containers": 3
+}
+```
+
+Alternatively, you can specify the entire "components" section instead.
+
+PUT URL - http://localhost:8088/app/v1/services/hello-world
+##### PUT Request JSON
+```json
+{
+ "state": "FLEX",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 3
+ }
+ ]
+}
+```
+
+### Destroy a service
+DELETE URL - http://localhost:8088/app/v1/services/hello-world
+
+***
+
+### Create a complicated service - HBase
+POST URL - http://localhost:8088:/app/v1/services/hbase-app-1
+
+##### POST Request JSON
+
+```json
+{
+ "name": "hbase-app-1",
+ "version": "1.0.0",
+ "description": "hbase service",
+ "lifetime": "3600",
+ "components": [
+ {
+ "name": "hbasemaster",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "properties": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "properties": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010"
+ }
+ }
+ ]
+ }
+ },
+ {
+ "name": "regionserver",
+ "number_of_containers": 3,
+ "artifact": {
+ "id": "hbase:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
+ "resource": {
+ "cpus": 1,
+ "memory": "2048"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_DIR": "<LOG_DIR>"
+ },
+ "files": [
+ {
+ "type": "XML",
+ "dest_file": "/etc/hadoop/conf/core-site.xml",
+ "properties": {
+ "fs.defaultFS": "${CLUSTER_FS_URI}"
+ }
+ },
+ {
+ "type": "XML",
+ "dest_file": "/etc/hbase/conf/hbase-site.xml",
+ "properties": {
+ "hbase.cluster.distributed": "true",
+ "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
+ "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
+ "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
+ "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
+ "hbase.master.info.port": "16010",
+ "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "quicklinks": {
+ "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
+ "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
+ }
+}
+```
+
+### Create a service requesting GPUs in addition to CPUs and RAM
+POST URL - http://localhost:8088/app/v1/services
+
+##### POST Request JSON
+```json
+{
+ "name": "hello-world",
+ "version": "1.0.0",
+ "description": "hello world example with GPUs",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 2,
+ "artifact": {
+ "id": "nginx:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "./start_nginx.sh",
+ "resource": {
+ "cpus": 1,
+ "memory": "256",
+ "additional" : {
+ "yarn.io/gpu" : {
+ "value" : 4,
+ "unit" : ""
+ }
+ }
+ }
+ }
+ ]
+}
+```
+
+### Create a service with a component requesting anti-affinity placement policy
+POST URL - http://localhost:8088/app/v1/services
+
+##### POST Request JSON
+```json
+{
+ "name": "hello-world",
+ "version": "1.0.0",
+ "description": "hello world example with anti-affinity",
+ "components" :
+ [
+ {
+ "name": "hello",
+ "number_of_containers": 3,
+ "artifact": {
+ "id": "nginx:latest",
+ "type": "DOCKER"
+ },
+ "launch_command": "./start_nginx.sh",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "placement_policy": {
+ "constraints": [
+ {
+ "type": "ANTI_AFFINITY",
+ "scope": "NODE",
+ "node_attributes": {
+ "os": ["linux", "windows"],
+ "fault_domain": ["fd1", "fd2"]
+ },
+ "node_partitions": [
+ "gpu",
+ "fast-disk"
+ ],
+ "target_tags": [
+ "hello"
+ ]
+ }
+ ]
+ }
+ }
+ ]
+}
+```
+
+##### GET Response JSON
+GET URL - http://localhost:8088/app/v1/services/hello-world
+
+Note, for an anti-affinity component no more than 1 container will be allocated
+in a specific node. In this example, 3 containers have been requested by
+component "hello". All 3 containers were allocated because the cluster had 3 or
+more NMs. If the cluster had less than 3 NMs then less than 3 containers would
+be allocated. In cases when the number of allocated containers are less than the
+number of requested containers, the component and the service will be in
+non-STABLE state.
+
+```json
+{
+ "name": "hello-world",
+ "version": "1.0.0",
+ "description": "hello world example with anti-affinity",
+ "id": "application_1503963985568_0003",
+ "lifetime": -1,
+ "state": "STABLE",
+ "components": [
+ {
+ "name": "hello",
+ "state": "STABLE",
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "placement_policy": {
+ "constraints": [
+ {
+ "type": "ANTI_AFFINITY",
+ "scope": "NODE",
+ "node_attributes": {
+ "os": ["linux", "windows"],
+ "fault_domain": ["fd1", "fd2"]
+ },
+ "node_partitions": [
+ "gpu",
+ "fast-disk"
+ ],
+ "target_tags": [
+ "hello"
+ ]
+ }
+ ]
+ },
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": [],
+ "containers": [
+ {
+ "id": "container_e03_1503963985568_0003_01_000002",
+ "ip": "10.22.8.143",
+ "hostname": "ctr-e03-1503963985568-0003-01-000002.example.site",
+ "state": "READY",
+ "launch_time": 1504051512412,
+ "bare_host": "host100.cloud.com",
+ "component_instance_name": "hello-0"
+ },
+ {
+ "id": "container_e03_1503963985568_0003_01_000003",
+ "ip": "10.22.8.144",
+ "hostname": "ctr-e03-1503963985568-0003-01-000003.example.site",
+ "state": "READY",
+ "launch_time": 1504051536450,
+ "bare_host": "host101.cloud.com",
+ "component_instance_name": "hello-1"
+ },
+ {
+ "id": "container_e03_1503963985568_0003_01_000004",
+ "ip": "10.22.8.145",
+ "hostname": "ctr-e03-1503963985568-0003-01-000004.example.site",
+ "state": "READY",
+ "launch_time": 1504051536450,
+ "bare_host": "host102.cloud.com",
+ "component_instance_name": "hello-2"
+ }
+ ],
+ "launch_command": "./start_nginx.sh",
+ "number_of_containers": 1,
+ "run_privileged_container": false
+ }
+ ],
+ "configuration": {
+ "properties": {},
+ "env": {},
+ "files": []
+ },
+ "quicklinks": {}
+}
+```
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[5/6] hadoop git commit: YARN-7530. Refactored YARN service API
project location. Contributed by Chandni Singh
Posted by ey...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
deleted file mode 100644
index f4acd94..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.webapp;
-
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.AuthenticationFilterInitializer;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.eclipse.jetty.webapp.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
-
-/**
- * This class launches the web service using Hadoop HttpServer2 (which uses
- * an embedded Jetty container). This is the entry point to your service.
- * The Java command used to launch this app should call the main method.
- */
-public class ApiServerWebApp extends AbstractService {
- private static final Logger logger = LoggerFactory
- .getLogger(ApiServerWebApp.class);
- private static final String SEP = ";";
-
- // REST API server for YARN native services
- private HttpServer2 apiServer;
- private InetSocketAddress bindAddress;
-
- public static void main(String[] args) throws IOException {
- ApiServerWebApp apiWebApp = new ApiServerWebApp();
- try {
- apiWebApp.init(new YarnConfiguration());
- apiWebApp.serviceStart();
- } catch (Exception e) {
- logger.error("Got exception starting", e);
- apiWebApp.close();
- }
- }
-
- public ApiServerWebApp() {
- super(ApiServerWebApp.class.getName());
- }
-
- @Override
- protected void serviceStart() throws Exception {
- bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS,
- DEFAULT_API_SERVER_ADDRESS, DEFAULT_API_SERVER_PORT);
- logger.info("YARN API server running on " + bindAddress);
- if (UserGroupInformation.isSecurityEnabled()) {
- doSecureLogin(getConfig());
- }
- startWebApp();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (apiServer != null) {
- apiServer.stop();
- }
- super.serviceStop();
- }
-
- private void doSecureLogin(org.apache.hadoop.conf.Configuration conf)
- throws IOException {
- SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB,
- YarnConfiguration.RM_PRINCIPAL, bindAddress.getHostName());
- addFilters(conf);
- }
-
- private void addFilters(org.apache.hadoop.conf.Configuration conf) {
- // Always load pseudo authentication filter to parse "user.name" in an URL
- // to identify a HTTP request's user.
- boolean hasHadoopAuthFilterInitializer = false;
- String filterInitializerConfKey = "hadoop.http.filter.initializers";
- Class<?>[] initializersClasses =
- conf.getClasses(filterInitializerConfKey);
- List<String> targets = new ArrayList<String>();
- if (initializersClasses != null) {
- for (Class<?> initializer : initializersClasses) {
- if (initializer.getName().equals(
- AuthenticationFilterInitializer.class.getName())) {
- hasHadoopAuthFilterInitializer = true;
- break;
- }
- targets.add(initializer.getName());
- }
- }
- if (!hasHadoopAuthFilterInitializer) {
- targets.add(AuthenticationFilterInitializer.class.getName());
- conf.set(filterInitializerConfKey, StringUtils.join(",", targets));
- }
- }
-
- private void startWebApp() throws IOException {
- URI uri = URI.create("http://" + NetUtils.getHostPortString(bindAddress));
-
- apiServer = new HttpServer2.Builder()
- .setName("api-server")
- .setConf(getConfig())
- .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
- .setUsernameConfKey(RM_WEBAPP_SPNEGO_USER_NAME_KEY)
- .setKeytabConfKey(RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
- .addEndpoint(uri).build();
-
- String apiPackages =
- ApiServer.class.getPackage().getName() + SEP
- + GenericExceptionHandler.class.getPackage().getName() + SEP
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
- apiServer.addJerseyResourcePackage(apiPackages, "/*");
-
- try {
- logger.info("Service starting up. Logging start...");
- apiServer.start();
- logger.info("Server status = {}", apiServer.toString());
- for (Configuration conf : apiServer.getWebAppContext()
- .getConfigurations()) {
- logger.info("Configurations = {}", conf);
- }
- logger.info("Context Path = {}", Collections.singletonList(
- apiServer.getWebAppContext().getContextPath()));
- logger.info("ResourceBase = {}", Collections.singletonList(
- apiServer.getWebAppContext().getResourceBase()));
- logger.info("War = {}", Collections
- .singletonList(apiServer.getWebAppContext().getWar()));
- } catch (Exception ex) {
- logger.error("Hadoop HttpServer2 App **failed**", ex);
- throw ex;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java
deleted file mode 100644
index 1bdf05a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.service.webapp contains classes to be used
- * for YARN Services API.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.service.webapp;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
deleted file mode 100644
index b7ad6c9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
+++ /dev/null
@@ -1,444 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-## Examples
-
-### Create a simple single-component service with most attribute values as defaults
-POST URL - http://localhost:8088/app/v1/services
-
-##### POST Request JSON
-```json
-{
- "name": "hello-world",
- "version": "1.0.0",
- "description": "hello world example",
- "components" :
- [
- {
- "name": "hello",
- "number_of_containers": 2,
- "artifact": {
- "id": "nginx:latest",
- "type": "DOCKER"
- },
- "launch_command": "./start_nginx.sh",
- "resource": {
- "cpus": 1,
- "memory": "256"
- }
- }
- ]
-}
-```
-
-##### GET Response JSON
-GET URL - http://localhost:8088/app/v1/services/hello-world
-
-Note, lifetime value of -1 means unlimited lifetime.
-
-```json
-{
- "name": "hello-world",
- "version": "1.0.0",
- "description": "hello world example",
- "id": "application_1503963985568_0002",
- "lifetime": -1,
- "state": "STABLE",
- "components": [
- {
- "name": "hello",
- "state": "STABLE",
- "resource": {
- "cpus": 1,
- "memory": "256"
- },
- "configuration": {
- "properties": {},
- "env": {},
- "files": []
- },
- "quicklinks": [],
- "containers": [
- {
- "id": "container_e03_1503963985568_0002_01_000002",
- "ip": "10.22.8.143",
- "hostname": "ctr-e03-1503963985568-0002-01-000002.example.site",
- "state": "READY",
- "launch_time": 1504051512412,
- "bare_host": "host100.cloud.com",
- "component_instance_name": "hello-0"
- },
- {
- "id": "container_e03_1503963985568_0002_01_000003",
- "ip": "10.22.8.144",
- "hostname": "ctr-e03-1503963985568-0002-01-000003.example.site",
- "state": "READY",
- "launch_time": 1504051536450,
- "bare_host": "host100.cloud.com",
- "component_instance_name": "hello-1"
- }
- ],
- "launch_command": "./start_nginx.sh",
- "number_of_containers": 1,
- "run_privileged_container": false
- }
- ],
- "configuration": {
- "properties": {},
- "env": {},
- "files": []
- },
- "quicklinks": {}
-}
-
-```
-### Update to modify the lifetime of a service
-PUT URL - http://localhost:8088/app/v1/services/hello-world
-
-##### PUT Request JSON
-
-Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
-
-```json
-{
- "lifetime": 3600
-}
-```
-### Stop a service
-PUT URL - http://localhost:8088/app/v1/services/hello-world
-
-##### PUT Request JSON
-```json
-{
- "state": "STOPPED"
-}
-```
-
-### Start a service
-PUT URL - http://localhost:8088/app/v1/services/hello-world
-
-##### PUT Request JSON
-```json
-{
- "state": "STARTED"
-}
-```
-
-### Update to flex up/down the number of containers (instances) of a component of a service
-PUT URL - http://localhost:8088/app/v1/services/hello-world/components/hello
-
-##### PUT Request JSON
-```json
-{
- "number_of_containers": 3
-}
-```
-
-Alternatively, you can specify the entire "components" section instead.
-
-PUT URL - http://localhost:8088/app/v1/services/hello-world
-##### PUT Request JSON
-```json
-{
- "state": "FLEX",
- "components" :
- [
- {
- "name": "hello",
- "number_of_containers": 3
- }
- ]
-}
-```
-
-### Destroy a service
-DELETE URL - http://localhost:8088/app/v1/services/hello-world
-
-***
-
-### Create a complicated service - HBase
-POST URL - http://localhost:8088:/app/v1/services/hbase-app-1
-
-##### POST Request JSON
-
-```json
-{
- "name": "hbase-app-1",
- "version": "1.0.0",
- "description": "hbase service",
- "lifetime": "3600",
- "components": [
- {
- "name": "hbasemaster",
- "number_of_containers": 1,
- "artifact": {
- "id": "hbase:latest",
- "type": "DOCKER"
- },
- "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start",
- "resource": {
- "cpus": 1,
- "memory": "2048"
- },
- "configuration": {
- "env": {
- "HBASE_LOG_DIR": "<LOG_DIR>"
- },
- "files": [
- {
- "type": "XML",
- "dest_file": "/etc/hadoop/conf/core-site.xml",
- "properties": {
- "fs.defaultFS": "${CLUSTER_FS_URI}"
- }
- },
- {
- "type": "XML",
- "dest_file": "/etc/hbase/conf/hbase-site.xml",
- "properties": {
- "hbase.cluster.distributed": "true",
- "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
- "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
- "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
- "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
- "hbase.master.info.port": "16010"
- }
- }
- ]
- }
- },
- {
- "name": "regionserver",
- "number_of_containers": 3,
- "artifact": {
- "id": "hbase:latest",
- "type": "DOCKER"
- },
- "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start",
- "resource": {
- "cpus": 1,
- "memory": "2048"
- },
- "configuration": {
- "env": {
- "HBASE_LOG_DIR": "<LOG_DIR>"
- },
- "files": [
- {
- "type": "XML",
- "dest_file": "/etc/hadoop/conf/core-site.xml",
- "properties": {
- "fs.defaultFS": "${CLUSTER_FS_URI}"
- }
- },
- {
- "type": "XML",
- "dest_file": "/etc/hbase/conf/hbase-site.xml",
- "properties": {
- "hbase.cluster.distributed": "true",
- "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
- "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
- "zookeeper.znode.parent": "${SERVICE_ZK_PATH}",
- "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}",
- "hbase.master.info.port": "16010",
- "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
- }
- }
- ]
- }
- }
- ],
- "quicklinks": {
- "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status",
- "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
- }
-}
-```
-
-### Create a service requesting GPUs in addition to CPUs and RAM
-POST URL - http://localhost:8088/app/v1/services
-
-##### POST Request JSON
-```json
-{
- "name": "hello-world",
- "version": "1.0.0",
- "description": "hello world example with GPUs",
- "components" :
- [
- {
- "name": "hello",
- "number_of_containers": 2,
- "artifact": {
- "id": "nginx:latest",
- "type": "DOCKER"
- },
- "launch_command": "./start_nginx.sh",
- "resource": {
- "cpus": 1,
- "memory": "256",
- "additional" : {
- "yarn.io/gpu" : {
- "value" : 4,
- "unit" : ""
- }
- }
- }
- }
- ]
-}
-```
-
-### Create a service with a component requesting anti-affinity placement policy
-POST URL - http://localhost:8088/app/v1/services
-
-##### POST Request JSON
-```json
-{
- "name": "hello-world",
- "version": "1.0.0",
- "description": "hello world example with anti-affinity",
- "components" :
- [
- {
- "name": "hello",
- "number_of_containers": 3,
- "artifact": {
- "id": "nginx:latest",
- "type": "DOCKER"
- },
- "launch_command": "./start_nginx.sh",
- "resource": {
- "cpus": 1,
- "memory": "256"
- },
- "placement_policy": {
- "constraints": [
- {
- "type": "ANTI_AFFINITY",
- "scope": "NODE",
- "node_attributes": {
- "os": ["linux", "windows"],
- "fault_domain": ["fd1", "fd2"]
- },
- "node_partitions": [
- "gpu",
- "fast-disk"
- ],
- "target_tags": [
- "hello"
- ]
- }
- ]
- }
- }
- ]
-}
-```
-
-##### GET Response JSON
-GET URL - http://localhost:8088/app/v1/services/hello-world
-
-Note, for an anti-affinity component no more than 1 container will be allocated
-in a specific node. In this example, 3 containers have been requested by
-component "hello". All 3 containers were allocated because the cluster had 3 or
-more NMs. If the cluster had less than 3 NMs then less than 3 containers would
-be allocated. In cases when the number of allocated containers are less than the
-number of requested containers, the component and the service will be in
-non-STABLE state.
-
-```json
-{
- "name": "hello-world",
- "version": "1.0.0",
- "description": "hello world example with anti-affinity",
- "id": "application_1503963985568_0003",
- "lifetime": -1,
- "state": "STABLE",
- "components": [
- {
- "name": "hello",
- "state": "STABLE",
- "resource": {
- "cpus": 1,
- "memory": "256"
- },
- "placement_policy": {
- "constraints": [
- {
- "type": "ANTI_AFFINITY",
- "scope": "NODE",
- "node_attributes": {
- "os": ["linux", "windows"],
- "fault_domain": ["fd1", "fd2"]
- },
- "node_partitions": [
- "gpu",
- "fast-disk"
- ],
- "target_tags": [
- "hello"
- ]
- }
- ]
- },
- "configuration": {
- "properties": {},
- "env": {},
- "files": []
- },
- "quicklinks": [],
- "containers": [
- {
- "id": "container_e03_1503963985568_0003_01_000002",
- "ip": "10.22.8.143",
- "hostname": "ctr-e03-1503963985568-0003-01-000002.example.site",
- "state": "READY",
- "launch_time": 1504051512412,
- "bare_host": "host100.cloud.com",
- "component_instance_name": "hello-0"
- },
- {
- "id": "container_e03_1503963985568_0003_01_000003",
- "ip": "10.22.8.144",
- "hostname": "ctr-e03-1503963985568-0003-01-000003.example.site",
- "state": "READY",
- "launch_time": 1504051536450,
- "bare_host": "host101.cloud.com",
- "component_instance_name": "hello-1"
- },
- {
- "id": "container_e03_1503963985568_0003_01_000004",
- "ip": "10.22.8.145",
- "hostname": "ctr-e03-1503963985568-0003-01-000004.example.site",
- "state": "READY",
- "launch_time": 1504051536450,
- "bare_host": "host102.cloud.com",
- "component_instance_name": "hello-2"
- }
- ],
- "launch_command": "./start_nginx.sh",
- "number_of_containers": 1,
- "run_privileged_container": false
- }
- ],
- "configuration": {
- "properties": {},
- "env": {},
- "files": []
- },
- "quicklinks": {}
-}
-```
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
deleted file mode 100644
index d90ae06..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ /dev/null
@@ -1,594 +0,0 @@
-# Hadoop YARN REST APIs for services v1 spec in YAML
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-swagger: '2.0'
-info:
- title: "YARN Simplified API layer for services"
- description: |
- Bringing a new service on YARN today is not a simple experience. The APIs of existing
- frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs)
- or writing a complex spec (for declarative frameworks).
-
- This simplified REST API can be used to create and manage the lifecycle of YARN services.
- In most cases, the application owner will not be forced to make any changes to their applications.
- This is primarily true if the application is packaged with containerization technologies like Docker.
-
- This document describes the API specifications (aka. YarnFile) for deploying/managing
- containerized services on YARN. The same JSON spec can be used for both REST API
- and CLI to manage the services.
-
- version: "1.0.0"
- license:
- name: Apache 2.0
- url: http://www.apache.org/licenses/LICENSE-2.0.html
-# the domain of the service
-host: localhost
-port: 8088(default)
-# array of all schemes that your API supports
-schemes:
- - http
-consumes:
- - application/json
-produces:
- - application/json
-paths:
- /app/v1/services/version:
- get:
- summary: Get current version of the API server.
- description: Get current version of the API server.
- responses:
- 200:
- description: Successful request
-
- /app/v1/services:
- get:
- summary: (TBD) List of services running in the cluster.
- description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
- responses:
- 200:
- description: An array of services
- schema:
- type: array
- items:
- $ref: '#/definitions/Service'
- default:
- description: Unexpected error
- schema:
- $ref: '#/definitions/ServiceStatus'
- post:
- summary: Create a service
- description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
- parameters:
- - name: Service
- in: body
- description: Service request object
- required: true
- schema:
- $ref: '#/definitions/Service'
- responses:
- 202:
- description: The request to create a service is accepted
- 400:
- description: Invalid service definition provided in the request body
- 500:
- description: Failed to create a service
- default:
- description: Unexpected error
- schema:
- $ref: '#/definitions/ServiceStatus'
-
- /app/v1/services/{service_name}:
- put:
- summary: Update a service or upgrade the binary version of the components of a running service
- description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service.
- The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD).
- parameters:
- - name: service_name
- in: path
- description: Service name
- required: true
- type: string
- - name: Service
- in: body
- description: The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service
- required: true
- schema:
- $ref: '#/definitions/Service'
- responses:
- 204:
- description: Update or upgrade was successful
- 404:
- description: Service does not exist
- default:
- description: Unexpected error
- schema:
- $ref: '#/definitions/ServiceStatus'
- delete:
- summary: Destroy a service
- description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc.
- parameters:
- - name: service_name
- in: path
- description: Service name
- required: true
- type: string
- responses:
- 204:
- description: Destroy was successful
- 404:
- description: Service does not exist
- default:
- description: Unexpected error
- schema:
- $ref: '#/definitions/ServiceStatus'
- get:
- summary: Get details of a service.
- description: Return the details (including containers) of a running service
- parameters:
- - name: service_name
- in: path
- description: Service name
- required: true
- type: string
- responses:
- 200:
- description: a service object
- schema:
- type: object
- items:
- $ref: '#/definitions/Service'
- examples:
- service_name: logsearch
- artifact:
- id: logsearch:latest
- type: docker
- 404:
- description: Service does not exist
- default:
- description: Unexpected error
- schema:
- $ref: '#/definitions/ServiceStatus'
- /app/v1/services/{service_name}/components/{component_name}:
- put:
- summary: Flex a component's number of instances.
- description: Set a component's desired number of instanes
- parameters:
- - name: service_name
- in: path
- description: Service name
- required: true
- type: string
- - name: component_name
- in: path
- description: Component name
- required: true
- type: string
- - name: Component
- in: body
- description: The definition of a component which contains the updated number of instances.
- required: true
- schema:
- $ref: '#/definitions/Component'
- responses:
- 200:
- description: Flex was successful
- 404:
- description: Service does not exist
- default:
- description: Unexpected error
- schema:
- $ref: '#/definitions/ServiceStatus'
-definitions:
- Service:
- description: a service resource has the following attributes.
- required:
- - name
- - version
- properties:
- name:
- type: string
- description: A unique service name. If Registry DNS is enabled, the max length is 63 characters.
- version:
- type: string
- description: Version of the service.
- description:
- type: string
- description: Description of the service.
- id:
- type: string
- description: A unique service id.
- artifact:
- description: The default artifact for all components of the service except the components which has Artifact type set to SERVICE (optional).
- $ref: '#/definitions/Artifact'
- resource:
- description: The default resource for all components of the service (optional).
- $ref: '#/definitions/Resource'
- launch_time:
- type: string
- format: date
- description: The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
- number_of_running_containers:
- type: integer
- format: int64
- description: In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.
- lifetime:
- type: integer
- format: int64
- description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
- components:
- description: Components of a service.
- type: array
- items:
- $ref: '#/definitions/Component'
- configuration:
- description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.
- $ref: '#/definitions/Configuration'
- state:
- description: State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state.
- $ref: '#/definitions/ServiceState'
- quicklinks:
- type: object
- description: A blob of key-value pairs of quicklinks to be exported for a service.
- additionalProperties:
- type: string
- queue:
- type: string
- description: The YARN queue that this service should be submitted to.
- kerberos_principal:
- description: The principal info of the user who launches the service.
- $ref: '#/definitions/KerberosPrincipal'
- docker_client_config:
- type: string
- description: URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json).
- ResourceInformation:
- description:
- ResourceInformation determines unit/value of resource types in addition to memory and vcores. It will be part of Resource object.
- properties:
- value:
- type: integer
- format: int64
- description: Integer value of the resource.
- unit:
- type: string
- description: Unit of the resource, acceptable values are - p/n/u/m/k/M/G/T/P/Ki/Mi/Gi/Ti/Pi. By default it is empty means no unit.
- Resource:
- description:
- Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
- properties:
- profile:
- type: string
- description: Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.
- cpus:
- type: integer
- format: int32
- description: Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).
- memory:
- type: string
- description: Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.
- additional:
- type: object
- additionalProperties:
- $ref: '#/definitions/ResourceInformation'
- description: A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below.
- PlacementPolicy:
- description: Advanced placement policy of the components of a service.
- required:
- - constraints
- properties:
- constraints:
- description: Placement constraint details.
- type: array
- items:
- $ref: '#/definitions/PlacementConstraint'
- PlacementConstraint:
- description: Placement constraint details.
- required:
- - type
- - scope
- properties:
- name:
- description: An optional name associated to this constraint.
- type: string
- example: C1
- type:
- description: The type of placement.
- $ref: '#/definitions/PlacementType'
- scope:
- description: The scope of placement.
- $ref: '#/definitions/PlacementScope'
- target_tags:
- description: The name of the components that this component's placement policy is depending upon are added as target tags. So for affinity say, this component's containers are requesting to be placed on hosts where containers of the target tag component(s) are running on. Target tags can also contain the name of this component, in which case it implies that for anti-affinity say, no more than one container of this component can be placed on a host. Similarly, for cardinality, it would mean that containers of this component is requesting to be placed on hosts where at least minCardinality but no more than maxCardinality containers of the target tag component(s) are running.
- type: array
- items:
- type: string
- node_attributes:
- description: Node attributes are a set of key:value(s) pairs associated with nodes.
- type: object
- additionalProperties:
- type: array
- items:
- type: string
- node_partitions:
- description: Node partitions where the containers of this component can run.
- type: array
- items:
- type: string
- min_cardinality:
- type: integer
- format: int64
- description: When placement type is cardinality, the minimum number of containers of the depending component that a host should have, where containers of this component can be allocated on.
- example: 2
- max_cardinality:
- type: integer
- format: int64
- description: When placement type is cardinality, the maximum number of containers of the depending component that a host should have, where containers of this component can be allocated on.
- example: 3
- PlacementType:
- description: The type of placement - affinity/anti-affinity/affinity-with-cardinality with containers of another component or containers of the same component (self).
- properties:
- type:
- type: string
- enum:
- - AFFINITY
- - ANTI_AFFINITY
- - AFFINITY_WITH_CARDINALITY
- PlacementScope:
- description: The scope of placement for the containers of a component.
- properties:
- type:
- type: string
- enum:
- - NODE
- - RACK
- Artifact:
- description: Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
- required:
- - id
- properties:
- id:
- type: string
- description: Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.
- type:
- type: string
- description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).
- enum:
- - DOCKER
- - TARBALL
- - SERVICE
- default: DOCKER
- uri:
- type: string
- description: Artifact location to support multiple artifact stores (optional).
- Component:
- description: One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
- required:
- - name
- properties:
- name:
- type: string
- description: Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
- state:
- description: The state of the component
- $ref: "#/definitions/ComponentState"
- dependencies:
- type: array
- items:
- type: string
- description: An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.
- readiness_check:
- description: Readiness check for this component.
- $ref: '#/definitions/ReadinessCheck'
- artifact:
- description: Artifact of the component (optional). If not specified, the service level global artifact takes effect.
- $ref: '#/definitions/Artifact'
- launch_command:
- type: string
- description: The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).
- resource:
- description: Resource of this component (optional). If not specified, the service level global resource takes effect.
- $ref: '#/definitions/Resource'
- number_of_containers:
- type: integer
- format: int64
- description: Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.
- containers:
- type: array
- description: Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.
- items:
- $ref: '#/definitions/Container'
- run_privileged_container:
- type: boolean
- description: Run all containers of this component in privileged mode (YARN-4262).
- placement_policy:
- description: Advanced scheduling and placement policies for all containers of this component.
- $ref: '#/definitions/PlacementPolicy'
- configuration:
- description: Config properties for this component.
- $ref: '#/definitions/Configuration'
- quicklinks:
- type: array
- items:
- type: string
- description: A list of quicklink keys defined at the service level, and to be resolved by this component.
- restartPolicy:
- type: string
- description: Policy of restart component. Including ALWAYS (Always restart component even if instance exit code = 0); ON_FAILURE (Only restart component if instance exit code != 0); NEVER (Do not restart in any cases)
- enum:
- - ALWAYS
- - ON_FAILURE
- - NEVER
- default: ALWAYS
- ReadinessCheck:
- description: A check to be performed to determine the readiness of a component instance (a container). If no readiness check is specified, the default readiness check will be used unless the yarn.service.default-readiness-check.enabled configuration property is set to false at the component, service, or system level. The artifact field is currently unsupported but may be implemented in the future, enabling a pluggable helper container to support advanced use cases.
- required:
- - type
- properties:
- type:
- type: string
- description: DEFAULT (AM checks whether the container has an IP and optionally performs a DNS lookup for the container hostname), HTTP (AM performs default checks, plus sends a REST call to the container and expects a response code between 200 and 299), or PORT (AM performs default checks, plus attempts to open a socket connection to the container on a specified port).
- enum:
- - DEFAULT
- - HTTP
- - PORT
- properties:
- type: object
- description: A blob of key value pairs that will be used to configure the check.
- additionalProperties:
- type: string
- artifact:
- description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET
- $ref: '#/definitions/Artifact'
- Configuration:
- description: Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
- properties:
- properties:
- type: object
- description: A blob of key-value pairs for configuring the YARN service AM.
- additionalProperties:
- type: string
- env:
- type: object
- description: A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.
- additionalProperties:
- type: string
- files:
- description: Array of list of files that needs to be created and made available as volumes in the service component containers.
- type: array
- items:
- $ref: '#/definitions/ConfigFile'
- ConfigFile:
- description: A config file that needs to be created and made available as a volume in a service component container.
- properties:
- type:
- type: string
- description: Config file in the standard format like xml, properties, json, yaml, template.
- enum:
- - XML
- - PROPERTIES
- - JSON
- - YAML
- - TEMPLATE
- - HADOOP_XML
- - STATIC
- - ARCHIVE
- dest_file:
- type: string
- description: The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.
- src_file:
- type: string
- description: This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.
- properties:
- type: object
- description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.
- additionalProperties:
- type: string
- Container:
- description: An instance of a running service container.
- properties:
- id:
- type: string
- description: Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.
- launch_time:
- type: string
- format: date
- description: The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.
- ip:
- type: string
- description: IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.
- hostname:
- type: string
- description: Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.
- bare_host:
- type: string
- description: The bare node or host in which the container is running, e.g. cn008.example.com.
- state:
- description: State of the container of a service.
- $ref: '#/definitions/ContainerState'
- component_instance_name:
- type: string
- description: Name of the component instance that this container instance belongs to. Component instance name is named as $COMPONENT_NAME-i, where i is a
- monotonically increasing integer. E.g. A componet called nginx can have multiple component instances named as nginx-0, nginx-1 etc.
- Each component instance is backed by a container instance.
- resource:
- description: Resource used for this container.
- $ref: '#/definitions/Resource'
- artifact:
- description: Artifact used for this container.
- $ref: '#/definitions/Artifact'
- privileged_container:
- type: boolean
- description: Container running in privileged mode or not.
- ServiceState:
- description: The current state of a service.
- properties:
- state:
- type: string
- description: enum of the state of the service
- enum:
- - ACCEPTED
- - STARTED
- - STABLE
- - STOPPED
- - FAILED
- - FLEX
- - UPGRADING
- ContainerState:
- description: The current state of the container of a service.
- properties:
- state:
- type: string
- description: enum of the state of the container
- enum:
- - INIT
- - STARTED
- - READY
- ComponentState:
- description: The state of the component
- properties:
- state:
- type: string
- description: enum of the state of the component
- enum:
- - INIT
- - FLEXING
- - STABLE
- - UPGRADING
- ServiceStatus:
- description: The current status of a submitted service, returned as a response to the GET API.
- properties:
- diagnostics:
- type: string
- description: Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.
- state:
- description: Service state.
- $ref: '#/definitions/ServiceState'
- code:
- type: integer
- format: int32
- description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.
- KerberosPrincipal:
- description: The kerberos principal info of the user who launches the service.
- properties:
- principal_name:
- type: string
- description: The principal name of the user who launches the service. Note that `_HOST` is required in the `principal_name` field such as `testuser/_HOST@EXAMPLE.COM` because Hadoop client validates that the server's (in this case, the AM's) principal has hostname present when communicating to the server.
- keytab:
- type: string
- description: The URI of the kerberos keytab. Currently supports only files present on the bare host. URI starts with "file\://" - A path on the local host where the keytab is stored. It is assumed that admin pre-installs the keytabs on the local host before AM launches.
-
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
deleted file mode 100644
index 8c679b9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
+++ /dev/null
@@ -1,76 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is the log4j configuration for YARN Services REST API Server
-
-# Log rotation based on size (100KB) with a max of 10 backup files
-log4j.rootLogger=INFO, restservicelog
-log4j.threshhold=ALL
-
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
-
-log4j.appender.restservicelog=org.apache.log4j.RollingFileAppender
-log4j.appender.restservicelog.layout=org.apache.log4j.PatternLayout
-log4j.appender.restservicelog.File=${REST_SERVICE_LOG_DIR}/restservice.log
-log4j.appender.restservicelog.MaxFileSize=1GB
-log4j.appender.restservicelog.MaxBackupIndex=10
-
-# log layout skips stack-trace creation operations by avoiding line numbers and method
-log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
-
-# debug edition is much more expensive
-#log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# configure stderr
-# set the conversion pattern of stderr
-# Print the date in ISO 8601 format
-log4j.appender.stderr=org.apache.log4j.ConsoleAppender
-log4j.appender.stderr.Target=System.err
-log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
-
-log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
-log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
-log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
-
-# for debugging REST API Service
-#log4j.logger.org.apache.hadoop.yarn.services=DEBUG
-
-# uncomment to debug service lifecycle issues
-#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
-
-# uncomment for YARN operations
-#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
-
-# uncomment this to debug security problems
-#log4j.logger.org.apache.hadoop.security=DEBUG
-
-#crank back on some noise
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-log4j.logger.org.apache.hadoop.hdfs=WARN
-log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
-
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
-log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.org.apache.curator.framework.state=ERROR
-log4j.logger.org.apache.curator.framework.imps=WARN
-
-log4j.logger.org.mortbay.log=DEBUG
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app
deleted file mode 100644
index 6a077b1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DON'T DELETE. REST WEBAPP RUN SCRIPT WILL STOP WORKING.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
deleted file mode 100644
index 1282c9f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<web-app xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
- version="3.0">
-
- <servlet>
- <servlet-name>Jersey REST API</servlet-name>
- <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
- <init-param>
- <param-name>com.sun.jersey.config.property.packages</param-name>
- <param-value>org.apache.hadoop.yarn.service.webapp,org.apache.hadoop.yarn.service.api,org.apache.hadoop.yarn.service.api.records</param-value>
- </init-param>
- <init-param>
- <param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name>
- <param-value>true</param-value>
- </init-param>
- <load-on-startup>1</load-on-startup>
- </servlet>
- <servlet-mapping>
- <servlet-name>Jersey REST API</servlet-name>
- <url-pattern>/*</url-pattern>
- </servlet-mapping>
-</web-app>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ff8d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
deleted file mode 100644
index 75b9486..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * A mock version of ServiceClient - This class is design
- * to simulate various error conditions that will happen
- * when a consumer class calls ServiceClient.
- */
-public class ServiceClientTest extends ServiceClient {
-
- private Configuration conf = new Configuration();
- private Service goodServiceStatus = buildLiveGoodService();
- private boolean initialized;
- private Set<String> expectedInstances = new HashSet<>();
-
- public ServiceClientTest() {
- super();
- }
-
- @Override
- public void init(Configuration conf) {
- if (!initialized) {
- super.init(conf);
- initialized = true;
- }
- }
-
- @Override
- public void stop() {
- // This is needed for testing API Server which uses client to get status
- // and then perform an action.
- }
-
- public void forceStop() {
- expectedInstances.clear();
- super.stop();
- }
-
- @Override
- public Configuration getConfig() {
- return conf;
- }
-
- @Override
- public ApplicationId actionCreate(Service service) throws IOException {
- ServiceApiUtil.validateAndResolveService(service,
- new SliderFileSystem(conf), getConfig());
- return ApplicationId.newInstance(System.currentTimeMillis(), 1);
- }
-
- @Override
- public Service getStatus(String appName) throws FileNotFoundException {
- if ("jenkins".equals(appName)) {
- return goodServiceStatus;
- } else {
- throw new FileNotFoundException("Service " + appName + " not found");
- }
- }
-
- @Override
- public int actionStart(String serviceName)
- throws YarnException, IOException {
- if (serviceName != null && serviceName.equals("jenkins")) {
- return EXIT_SUCCESS;
- } else {
- throw new ApplicationNotFoundException("");
- }
- }
-
- @Override
- public int actionStop(String serviceName, boolean waitForAppStopped)
- throws YarnException, IOException {
- if (serviceName == null) {
- throw new NullPointerException();
- }
- if (serviceName.equals("jenkins")) {
- return EXIT_SUCCESS;
- } else if (serviceName.equals("jenkins-second-stop")) {
- return EXIT_COMMAND_ARGUMENT_ERROR;
- } else {
- throw new ApplicationNotFoundException("");
- }
- }
-
- @Override
- public int actionDestroy(String serviceName) {
- if (serviceName != null) {
- if (serviceName.equals("jenkins")) {
- return EXIT_SUCCESS;
- } else if (serviceName.equals("jenkins-already-stopped")) {
- return EXIT_SUCCESS;
- } else if (serviceName.equals("jenkins-doesn't-exist")) {
- return EXIT_NOT_FOUND;
- } else if (serviceName.equals("jenkins-error-cleaning-registry")) {
- return EXIT_OTHER_FAILURE;
- }
- }
- throw new IllegalArgumentException();
- }
-
- @Override
- public int initiateUpgrade(Service service) throws YarnException,
- IOException {
- if (service.getName() != null && service.getName().equals("jenkins")) {
- return EXIT_SUCCESS;
- } else {
- throw new IllegalArgumentException();
- }
- }
-
- @Override
- public int actionUpgrade(Service service, List<Container> compInstances)
- throws IOException, YarnException {
- if (service.getName() != null && service.getName().equals("jenkins")
- && compInstances != null) {
- Set<String> actualInstances = compInstances.stream().map(
- Container::getComponentInstanceName).collect(Collectors.toSet());
- if (actualInstances.equals(expectedInstances)) {
- return EXIT_SUCCESS;
- }
- }
- throw new IllegalArgumentException();
- }
-
- Service getGoodServiceStatus() {
- return goodServiceStatus;
- }
-
- void setExpectedInstances(Set<String> instances) {
- if (instances != null) {
- expectedInstances.addAll(instances);
- }
- }
-
- static Service buildGoodService() {
- Service service = new Service();
- service.setName("jenkins");
- service.setVersion("v1");
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.DOCKER);
- artifact.setId("jenkins:latest");
- Resource resource = new Resource();
- resource.setCpus(1);
- resource.setMemory("2048");
- List<Component> components = new ArrayList<>();
- for (int i = 0; i < 2; i++) {
- Component c = new Component();
- c.setName("jenkins" + i);
- c.setNumberOfContainers(2L);
- c.setArtifact(artifact);
- c.setLaunchCommand("");
- c.setResource(resource);
- components.add(c);
- }
- service.setComponents(components);
- return service;
- }
-
- static Service buildLiveGoodService() {
- Service service = buildGoodService();
- Component comp = service.getComponents().iterator().next();
- List<Container> containers = new ArrayList<>();
- for (int i = 0; i < comp.getNumberOfContainers(); i++) {
- Container container = new Container();
- container.setComponentInstanceName(comp.getName() + "-" + (i + 1));
- container.setState(ContainerState.READY);
- containers.add(container);
- }
- comp.setContainers(containers);
- return service;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org