You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/04/19 03:23:21 UTC
[09/28] hadoop git commit: YARN-8126. Support auto-spawning of admin
configured services during bootstrap of RM. Contributed by Rohith Sharma K S.
YARN-8126. Support auto-spawning of admin configured services during bootstrap of RM. Contributed by Rohith Sharma K S.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/427ad7ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/427ad7ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/427ad7ec
Branch: refs/heads/HDFS-7240
Commit: 427ad7ecc4afbf2cc4acde22c29a7248c46fb22b
Parents: c4d3636
Author: Sunil G <su...@apache.org>
Authored: Tue Apr 17 15:08:31 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Apr 17 15:08:31 2018 +0530
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 1 +
.../client/SystemServiceManagerImpl.java | 22 ++-
.../service/client/TestSystemServiceImpl.java | 180 ------------------
.../client/TestSystemServiceManagerImpl.java | 182 +++++++++++++++++++
.../resources/system-services/bad/bad.yarnfile | 16 ++
.../sync/user1/example-app1.yarnfile | 16 ++
.../sync/user1/example-app2.yarnfile | 16 ++
.../sync/user1/example-app3.json | 16 ++
.../sync/user2/example-app1.yarnfile | 16 ++
.../sync/user2/example-app2.yarnfile | 16 ++
.../users/sync/user1/example-app1.yarnfile | 16 --
.../users/sync/user1/example-app2.yarnfile | 16 --
.../users/sync/user1/example-app3.json | 16 --
.../users/sync/user2/example-app1.yarnfile | 16 --
.../users/sync/user2/example-app2.yarnfile | 16 --
.../yarn/service/TestSystemServiceManager.java | 156 ----------------
.../markdown/yarn-service/SystemServices.md | 66 +++++++
17 files changed, 361 insertions(+), 422 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index fdf5583..eda7183 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -167,6 +167,7 @@
<item name="Concepts" href="hadoop-yarn/hadoop-yarn-site/yarn-service/Concepts.html"/>
<item name="Yarn Service API" href="hadoop-yarn/hadoop-yarn-site/yarn-service/YarnServiceAPI.html"/>
<item name="Service Discovery" href="hadoop-yarn/hadoop-yarn-site/yarn-service/ServiceDiscovery.html"/>
+ <item name="System Services" href="hadoop-yarn/hadoop-yarn-site/yarn-service/SystemServices.html"/>
</menu>
<menu name="Hadoop Compatible File Systems" inherit="top">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
index 225f8bd..f9cfa92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
@@ -92,10 +92,12 @@ public class SystemServiceManagerImpl extends AbstractService
private Thread serviceLaucher;
@VisibleForTesting
- private int skipCounter;
+ private int badFileNameExtensionSkipCounter;
@VisibleForTesting
private Map<String, Integer> ignoredUserServices =
new HashMap<>();
+ @VisibleForTesting
+ private int badDirSkipCounter;
public SystemServiceManagerImpl() {
super(SystemServiceManagerImpl.class.getName());
@@ -268,6 +270,7 @@ public class SystemServiceManagerImpl extends AbstractService
} else if (launchType.getPath().getName().equals(ASYNC)) {
scanForUserServiceDefinition(launchType.getPath(), asyncUserServices);
} else {
+ badDirSkipCounter++;
LOG.debug("Scanner skips for unknown dir {}.", launchType.getPath());
}
}
@@ -308,7 +311,7 @@ public class SystemServiceManagerImpl extends AbstractService
if (!filename.endsWith(YARN_FILE_SUFFIX)) {
LOG.info("Scanner skips for unknown file extension, filename = {}",
filename);
- skipCounter++;
+ badFileNameExtensionSkipCounter++;
continue;
}
Service service = getServiceDefinition(serviceCache.getPath());
@@ -325,9 +328,10 @@ public class SystemServiceManagerImpl extends AbstractService
LOG.warn(
"Ignoring service {} for the user {} as it is already present,"
+ " filename = {}", service.getName(), userName, filename);
+ } else {
+ LOG.info("Added service {} for the user {}, filename = {}",
+ service.getName(), userName, filename);
}
- LOG.info("Added service {} for the user {}, filename = {}",
- service.getName(), userName, filename);
}
}
}
@@ -375,7 +379,13 @@ public class SystemServiceManagerImpl extends AbstractService
return syncUserServices;
}
- @VisibleForTesting int getSkipCounter() {
- return skipCounter;
+ @VisibleForTesting
+ int getBadFileNameExtensionSkipCounter() {
+ return badFileNameExtensionSkipCounter;
+ }
+
+ @VisibleForTesting
+ int getBadDirSkipCounter() {
+ return badDirSkipCounter;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceImpl.java
deleted file mode 100644
index 27632f9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceImpl.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Test class for system service manager.
- */
-public class TestSystemServiceImpl {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestSystemServiceImpl.class);
- private SystemServiceManagerImpl systemService;
- private Configuration conf;
- private String resourcePath = "users";
-
- private String[] users = new String[] {"user1", "user2"};
- private static Map<String, Set<String>> loadedServices = new HashMap<>();
- private static Map<String, Set<String>> submittedServices = new HashMap<>();
-
- @Before
- public void setup() {
- File file = new File(
- getClass().getClassLoader().getResource(resourcePath).getFile());
- conf = new Configuration();
- conf.set(YarnServiceConf.YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY,
- file.getAbsolutePath());
- systemService = new SystemServiceManagerImpl() {
- @Override ServiceClient getServiceClient() {
- return new TestServiceClient();
- }
- };
- systemService.init(conf); // do not call explicit start
-
- constructUserService(users[0], "example-app1");
- constructUserService(users[1], "example-app1", "example-app2");
- }
-
- @After
- public void teadDown() {
- systemService.stop();
- }
-
- @Test
- public void testSystemServiceSubmission() throws Exception {
- systemService.start();
-
- /* verify for ignored sevices count */
- Map<String, Integer> ignoredUserServices =
- systemService.getIgnoredUserServices();
- Assert.assertEquals(1, ignoredUserServices.size());
- Assert.assertTrue("User user1 doesn't exist.",
- ignoredUserServices.containsKey(users[0]));
- int count = ignoredUserServices.get(users[0]);
- Assert.assertEquals(1, count);
- Assert.assertEquals(1, systemService.getSkipCounter());
-
- Map<String, Set<Service>> userServices =
- systemService.getSyncUserServices();
- Assert.assertEquals(loadedServices.size(), userServices.size());
- verifyForScannedUserServices(userServices);
-
- verifyForLaunchedUserServices();
-
- // 2nd time launch service to handle if service exist scenario
- systemService.launchUserService(userServices);
- verifyForLaunchedUserServices();
- }
-
- private void verifyForScannedUserServices(
- Map<String, Set<Service>> userServices) {
- for (String user : users) {
- Set<Service> services = userServices.get(user);
- Set<String> serviceNames = loadedServices.get(user);
- Assert.assertEquals(serviceNames.size(), services.size());
- Iterator<Service> iterator = services.iterator();
- while (iterator.hasNext()) {
- Service next = iterator.next();
- Assert.assertTrue(
- "Service name doesn't exist in expected " + "userService "
- + serviceNames, serviceNames.contains(next.getName()));
- }
- }
- }
-
- public void constructUserService(String user, String... serviceNames) {
- Set<String> service = loadedServices.get(user);
- if (service == null) {
- service = new HashSet<>();
- for (String serviceName : serviceNames) {
- service.add(serviceName);
- }
- loadedServices.put(user, service);
- }
- }
-
- class TestServiceClient extends ServiceClient {
- @Override
- protected void serviceStart() throws Exception {
- // do nothing
- }
-
- @Override
- protected void serviceStop() throws Exception {
- // do nothing
- }
-
- @Override
- protected void serviceInit(Configuration configuration)
- throws Exception {
- // do nothing
- }
-
- @Override
- public ApplicationId actionCreate(Service service)
- throws YarnException, IOException {
- String userName =
- UserGroupInformation.getCurrentUser().getShortUserName();
- Set<String> services = submittedServices.get(userName);
- if (services == null) {
- services = new HashSet<>();
- submittedServices.put(userName, services);
- }
- if (services.contains(service.getName())) {
- String message = "Failed to create service " + service.getName()
- + ", because it already exists.";
- throw new YarnException(message);
- }
- services.add(service.getName());
- return ApplicationId.newInstance(System.currentTimeMillis(), 1);
- }
- }
-
- private void verifyForLaunchedUserServices() {
- Assert.assertEquals(loadedServices.size(), submittedServices.size());
- for (Map.Entry<String, Set<String>> entry : submittedServices.entrySet()) {
- String user = entry.getKey();
- Set<String> serviceSet = entry.getValue();
- Assert.assertTrue(loadedServices.containsKey(user));
- Set<String> services = loadedServices.get(user);
- Assert.assertEquals(services.size(), serviceSet.size());
- Assert.assertTrue(services.containsAll(serviceSet));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
new file mode 100644
index 0000000..d39083d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Test class for system service manager.
+ */
+public class TestSystemServiceManagerImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestSystemServiceManagerImpl.class);
+ private SystemServiceManagerImpl systemService;
+ private Configuration conf;
+ private String resourcePath = "system-services";
+
+ private String[] users = new String[] {"user1", "user2"};
+ private static Map<String, Set<String>> loadedServices = new HashMap<>();
+ private static Map<String, Set<String>> submittedServices = new HashMap<>();
+
+ @Before
+ public void setup() {
+ File file = new File(
+ getClass().getClassLoader().getResource(resourcePath).getFile());
+ conf = new Configuration();
+ conf.set(YarnServiceConf.YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY,
+ file.getAbsolutePath());
+ systemService = new SystemServiceManagerImpl() {
+ @Override ServiceClient getServiceClient() {
+ return new TestServiceClient();
+ }
+ };
+ systemService.init(conf); // do not call explicit start
+
+ constructUserService(users[0], "example-app1");
+ constructUserService(users[1], "example-app1", "example-app2");
+ }
+
+ @After
+ public void teadDown() {
+ systemService.stop();
+ }
+
+ @Test
+ public void testSystemServiceSubmission() throws Exception {
+ systemService.start();
+
+ /* verify for ignored sevices count */
+ Map<String, Integer> ignoredUserServices =
+ systemService.getIgnoredUserServices();
+ Assert.assertEquals(1, ignoredUserServices.size());
+ Assert.assertTrue("User user1 doesn't exist.",
+ ignoredUserServices.containsKey(users[0]));
+ int count = ignoredUserServices.get(users[0]);
+ Assert.assertEquals(1, count);
+ Assert.assertEquals(1,
+ systemService.getBadFileNameExtensionSkipCounter());
+ Assert.assertEquals(1, systemService.getBadDirSkipCounter());
+
+ Map<String, Set<Service>> userServices =
+ systemService.getSyncUserServices();
+ Assert.assertEquals(loadedServices.size(), userServices.size());
+ verifyForScannedUserServices(userServices);
+
+ verifyForLaunchedUserServices();
+
+ // 2nd time launch service to handle if service exist scenario
+ systemService.launchUserService(userServices);
+ verifyForLaunchedUserServices();
+ }
+
+ private void verifyForScannedUserServices(
+ Map<String, Set<Service>> userServices) {
+ for (String user : users) {
+ Set<Service> services = userServices.get(user);
+ Set<String> serviceNames = loadedServices.get(user);
+ Assert.assertEquals(serviceNames.size(), services.size());
+ Iterator<Service> iterator = services.iterator();
+ while (iterator.hasNext()) {
+ Service next = iterator.next();
+ Assert.assertTrue(
+ "Service name doesn't exist in expected userService "
+ + serviceNames, serviceNames.contains(next.getName()));
+ }
+ }
+ }
+
+ public void constructUserService(String user, String... serviceNames) {
+ Set<String> service = loadedServices.get(user);
+ if (service == null) {
+ service = new HashSet<>();
+ for (String serviceName : serviceNames) {
+ service.add(serviceName);
+ }
+ loadedServices.put(user, service);
+ }
+ }
+
+ class TestServiceClient extends ServiceClient {
+ @Override
+ protected void serviceStart() throws Exception {
+ // do nothing
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ // do nothing
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration)
+ throws Exception {
+ // do nothing
+ }
+
+ @Override
+ public ApplicationId actionCreate(Service service)
+ throws YarnException, IOException {
+ String userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ Set<String> services = submittedServices.get(userName);
+ if (services == null) {
+ services = new HashSet<>();
+ submittedServices.put(userName, services);
+ }
+ if (services.contains(service.getName())) {
+ String message = "Failed to create service " + service.getName()
+ + ", because it already exists.";
+ throw new YarnException(message);
+ }
+ services.add(service.getName());
+ return ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ }
+ }
+
+ private void verifyForLaunchedUserServices() {
+ Assert.assertEquals(loadedServices.size(), submittedServices.size());
+ for (Map.Entry<String, Set<String>> entry : submittedServices.entrySet()) {
+ String user = entry.getKey();
+ Set<String> serviceSet = entry.getValue();
+ Assert.assertTrue(loadedServices.containsKey(user));
+ Set<String> services = loadedServices.get(user);
+ Assert.assertEquals(services.size(), serviceSet.size());
+ Assert.assertTrue(services.containsAll(serviceSet));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile
new file mode 100644
index 0000000..1d514d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "bad",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile
new file mode 100644
index 0000000..823561d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "example-app1",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile
new file mode 100644
index 0000000..823561d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "example-app1",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json
new file mode 100644
index 0000000..8a3a561
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json
@@ -0,0 +1,16 @@
+{
+ "name": "example-app3",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile
new file mode 100644
index 0000000..823561d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "example-app1",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile
new file mode 100644
index 0000000..d8fd1d1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile
@@ -0,0 +1,16 @@
+{
+ "name": "example-app2",
+ "version": "1.0.0",
+ "components" :
+ [
+ {
+ "name": "simple",
+ "number_of_containers": 1,
+ "launch_command": "sleep 2",
+ "resource": {
+ "cpus": 1,
+ "memory": "128"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app1.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app1.yarnfile
deleted file mode 100644
index 823561d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app1.yarnfile
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app1",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app2.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app2.yarnfile
deleted file mode 100644
index 823561d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app2.yarnfile
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app1",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app3.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app3.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app3.json
deleted file mode 100644
index 8a3a561..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app3.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app3",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app1.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app1.yarnfile
deleted file mode 100644
index 823561d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app1.yarnfile
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app1",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app2.yarnfile
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app2.yarnfile
deleted file mode 100644
index d8fd1d1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app2.yarnfile
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "example-app2",
- "version": "1.0.0",
- "components" :
- [
- {
- "name": "simple",
- "number_of_containers": 1,
- "launch_command": "sleep 2",
- "resource": {
- "cpus": 1,
- "memory": "128"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestSystemServiceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestSystemServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestSystemServiceManager.java
deleted file mode 100644
index dbff02f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestSystemServiceManager.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.ComponentState;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.api.records.ServiceState;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.mockito.Mockito.mock;
-
-/**
- * Tests for {@link ServiceManager}.
- */
-public class TestSystemServiceManager {
-
- @Rule
- public ServiceTestUtils.ServiceFSWatcher rule =
- new ServiceTestUtils.ServiceFSWatcher();
-
- @Test
- public void testUpgrade() throws IOException, SliderException {
- ServiceManager serviceManager = createTestServiceManager("testUpgrade");
- upgrade(serviceManager, "v2", false);
- Assert.assertEquals("service not upgraded", ServiceState.UPGRADING,
- serviceManager.getServiceSpec().getState());
- }
-
- @Test
- public void testRestartNothingToUpgrade()
- throws IOException, SliderException {
- ServiceManager serviceManager = createTestServiceManager("testRestart");
- upgrade(serviceManager, "v2", false);
-
- //make components stable
- serviceManager.getServiceSpec().getComponents().forEach(comp -> {
- comp.setState(ComponentState.STABLE);
- });
- serviceManager.handle(new ServiceEvent(ServiceEventType.START));
- Assert.assertEquals("service not re-started", ServiceState.STABLE,
- serviceManager.getServiceSpec().getState());
- }
-
- @Test
- public void testRestartWithPendingUpgrade()
- throws IOException, SliderException {
- ServiceManager serviceManager = createTestServiceManager("testRestart");
- upgrade(serviceManager, "v2", true);
- serviceManager.handle(new ServiceEvent(ServiceEventType.START));
- Assert.assertEquals("service should still be upgrading",
- ServiceState.UPGRADING, serviceManager.getServiceSpec().getState());
- }
-
-
- private void upgrade(ServiceManager service, String version,
- boolean upgradeArtifact)
- throws IOException, SliderException {
- Service upgradedDef = ServiceTestUtils.createExampleApplication();
- upgradedDef.setName(service.getName());
- upgradedDef.setVersion(version);
- if (upgradeArtifact) {
- Artifact upgradedArtifact = createTestArtifact("2");
- upgradedDef.getComponents().forEach(component -> {
- component.setArtifact(upgradedArtifact);
- });
- }
- writeUpgradedDef(upgradedDef);
- ServiceEvent upgradeEvent = new ServiceEvent(ServiceEventType.UPGRADE);
- upgradeEvent.setVersion("v2");
- service.handle(upgradeEvent);
- }
-
- private ServiceManager createTestServiceManager(String name)
- throws IOException {
- ServiceContext context = new ServiceContext();
- context.service = createBaseDef(name);
- context.fs = rule.getFs();
-
- context.scheduler = new ServiceScheduler(context) {
- @Override
- protected YarnRegistryViewForProviders createYarnRegistryOperations(
- ServiceContext context, RegistryOperations registryClient) {
- return mock(YarnRegistryViewForProviders.class);
- }
- };
-
- context.scheduler.init(rule.getConf());
-
- Map<String, org.apache.hadoop.yarn.service.component.Component>
- componentState = context.scheduler.getAllComponents();
- context.service.getComponents().forEach(component -> {
- componentState.put(component.getName(),
- new org.apache.hadoop.yarn.service.component.Component(component,
- 1L, context));
- });
- return new ServiceManager(context);
- }
-
- static Service createBaseDef(String name) {
- ApplicationId applicationId = ApplicationId.newInstance(
- System.currentTimeMillis(), 1);
- Service serviceDef = ServiceTestUtils.createExampleApplication();
- serviceDef.setId(applicationId.toString());
- serviceDef.setName(name);
- serviceDef.setState(ServiceState.STARTED);
- Artifact artifact = createTestArtifact("1");
-
- serviceDef.getComponents().forEach(component ->
- component.setArtifact(artifact));
- return serviceDef;
- }
-
- static Artifact createTestArtifact(String artifactId) {
- Artifact artifact = new Artifact();
- artifact.setId(artifactId);
- artifact.setType(Artifact.TypeEnum.TARBALL);
- return artifact;
- }
-
- private void writeUpgradedDef(Service upgradedDef)
- throws IOException, SliderException {
- Path upgradePath = rule.getFs().buildClusterUpgradeDirPath(
- upgradedDef.getName(), upgradedDef.getVersion());
- ServiceApiUtil.createDirAndPersistApp(rule.getFs(), upgradePath,
- upgradedDef);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/427ad7ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/SystemServices.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/SystemServices.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/SystemServices.md
new file mode 100644
index 0000000..20fed15
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/SystemServices.md
@@ -0,0 +1,66 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# System Services
+
+## Overview
+System services are admin configured services which are auto deployed during bootstrap of ResourceManager. This would work only when API-Server is started as part of ResourceManager. Refer [Manage services on YARN](QuickStart.html#Manage_services_on_YARN_via_REST_API). This document describes how to configure and deploy system services.
+
+## Configuration
+
+| Name | Description |
+| ------------ | ------------- |
+|yarn.service.system-service.dir| FS directory path to load and deploy admin configured services. These service spec files should be kept with proper hierarchy.|
+
+## Hierarchy of FS path
+After configuring *yarn.service.system-service.dir* path, the spec files should be kept with below hierarchy.
+````
+$SYSTEM_SERVICE_DIR_PATH/<Launch-Mode>/<Users>/<Yarnfiles>.
+````
+### Launch-Mode
+Launch-Mode indicates that how the service should be deployed. Services can be auto deployed either synchronously or asynchronously.
+
+#### sync
+These services are started synchronously along with RM. This might delay a bit RM transition to active period. This is useful when deploying critical services to get started sooner.
+
+#### async
+These services are started asynchronously without impacting RM transition period.
+
+### Users
+Users are the owner of the system service who has full access to modify it. Each users can own multiple services. Note that service names are unique per user.
+
+### Yarnfiles
+YarnFiles are the spec files to launch services. These files must have .yarnfile extension otherwise those files are ignored.
+
+### Example of hierarchy to configure system services.
+
+```
+SYSTEM_SERVICE_DIR_PATH
+|---- sync
+| |--- user1
+| | |---- service1.yarnfile
+| | |---- service2.yarnfile
+| |--- user2
+| | |---- service3.yarnfile
+| | ....
+| |
+|---- async
+| |--- user3
+| | |---- service1.yarnfile
+| | |---- service2.yarnfile
+| |--- user4
+| | |---- service3.yarnfile
+| | ....
+| |
+```
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org