You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2018/09/28 16:42:19 UTC
hadoop git commit: YARN-8734. Readiness check for remote service
belonging to the same user. Contributed by Eric Yang
Repository: hadoop
Updated Branches:
refs/heads/trunk e2113500d -> d1c1dde30
YARN-8734. Readiness check for remote service belonging to the same user. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1c1dde3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1c1dde3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1c1dde3
Branch: refs/heads/trunk
Commit: d1c1dde3097eea6a46a39d0312f94949f6884002
Parents: e211350
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Sep 28 09:39:59 2018 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Fri Sep 28 09:40:48 2018 -0700
----------------------------------------------------------------------
...RN-Simplified-V1-API-Layer-For-Services.yaml | 5 +++
.../hadoop/yarn/service/ServiceManager.java | 3 +-
.../hadoop/yarn/service/ServiceScheduler.java | 2 +
.../yarn/service/api/records/Service.java | 15 +++++++
.../yarn/service/utils/ServiceApiUtil.java | 43 ++++++++++++++++++++
.../yarn/service/TestYarnNativeServices.java | 2 +
.../yarn/service/utils/TestServiceApiUtil.java | 39 ++++++++++++++++++
.../markdown/yarn-service/YarnServiceAPI.md | 2 +-
8 files changed, 109 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index d90ae06..d74604a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -254,6 +254,11 @@ definitions:
docker_client_config:
type: string
description: URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json).
+ dependencies:
+ type: array
+ items:
+ type: string
+ description: An array of services which should be in STABLE state, before this service can be started.
ResourceInformation:
description:
ResourceInformation determines unit/value of resource types in addition to memory and vcores. It will be part of Resource object.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
index 4851325..aefdadd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
@@ -170,7 +170,8 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
} else {
serviceManager.setServiceState(ServiceState.UPGRADING);
}
-
+ ServiceApiUtil.checkServiceDependencySatisified(serviceManager
+ .getServiceSpec());
return State.UPGRADING;
} catch (Throwable e) {
LOG.error("[SERVICE]: Upgrade to version {} failed", event.getVersion(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index b49ef2a..9b9305c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -389,6 +389,8 @@ public class ServiceScheduler extends CompositeService {
// Since AM has been started and registered, the service is in STARTED state
app.setState(ServiceState.STARTED);
+ ServiceApiUtil.checkServiceDependencySatisified(context.service);
+
// recover components based on containers sent from RM
recoverComponents(response);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
index 57d1a1a..6e03121 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
@@ -73,6 +73,7 @@ public class Service extends BaseResource {
private String version = null;
private String description = null;
private String dockerClientConfig = null;
+ private List<String> dependencies = new ArrayList<String>();
/**
* A unique service name.
@@ -352,6 +353,18 @@ public class Service extends BaseResource {
this.queue = queue;
}
+ @ApiModelProperty(example = "null", value = "A list of dependent services.")
+ @XmlElement(name = "dependencies")
+ @JsonProperty("dependencies")
+ public List<String> getDependencies() {
+ return dependencies;
+ }
+
+ public void setDependencies(List<String>
+ dependencies) {
+ this.dependencies = dependencies;
+ }
+
public Service kerberosPrincipal(KerberosPrincipal kerberosPrincipal) {
this.kerberosPrincipal = kerberosPrincipal;
return this;
@@ -437,6 +450,8 @@ public class Service extends BaseResource {
.append(toIndentedString(kerberosPrincipal)).append("\n");
sb.append(" dockerClientConfig: ")
.append(toIndentedString(dockerClientConfig)).append("\n");
+ sb.append(" dependencies: ")
+ .append(toIndentedString(dependencies)).append("\n");
sb.append("}");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 0eb54ce..d201c7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.service.api.records.ComponentState;
import org.apache.hadoop.yarn.service.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Configuration;
@@ -705,4 +707,45 @@ public class ServiceApiUtil {
}
return components;
}
+
+ private static boolean serviceDependencySatisfied(Service service) {
+ boolean result = true;
+ try {
+ List<String> dependencies = service
+ .getDependencies();
+ org.apache.hadoop.conf.Configuration conf =
+ new org.apache.hadoop.conf.Configuration();
+ if (dependencies != null && dependencies.size() > 0) {
+ ServiceClient sc = new ServiceClient();
+ sc.init(conf);
+ sc.start();
+ for (String dependent : dependencies) {
+ Service dependentService = sc.getStatus(dependent);
+ if (dependentService.getState() == null ||
+ !dependentService.getState().equals(ServiceState.STABLE)) {
+ result = false;
+ LOG.info("Service dependency is not satisfied for " +
+ "service: {} state: {}", dependent,
+ dependentService.getState());
+ }
+ }
+ sc.close();
+ }
+ } catch (IOException | YarnException e) {
+ LOG.warn("Caught exception: ", e);
+ LOG.info("Service dependency is not satisified.");
+ result = false;
+ }
+ return result;
+ }
+
+ public static void checkServiceDependencySatisified(Service service) {
+ while (!serviceDependencySatisfied(service)) {
+ try {
+ LOG.info("Waiting for service dependencies.");
+ Thread.sleep(15000L);
+ } catch (InterruptedException e) {
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 3e23a10..d50c574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -438,6 +438,8 @@ public class TestYarnNativeServices extends ServiceTestUtils {
// wait for upgrade to complete
waitForServiceToBeStable(client, service);
Service active = client.getStatus(service.getName());
+ Assert.assertEquals("version mismatch", service.getVersion(),
+ active.getVersion());
Assert.assertEquals("component not stable", ComponentState.STABLE,
active.getComponent(component.getName()).getState());
Assert.assertEquals("compa does not have new env", "val1",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
index 98e7474..1e3c180 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.service.api.records.PlacementScope;
import org.apache.hadoop.yarn.service.api.records.PlacementType;
import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -733,6 +734,44 @@ public class TestServiceApiUtil extends ServiceTestUtils {
}
}
+ @Test(timeout = 1500)
+ public void testNoServiceDependencies() {
+ Service service = createExampleApplication();
+ Component compa = createComponent("compa");
+ Component compb = createComponent("compb");
+ service.addComponent(compa);
+ service.addComponent(compb);
+ List<String> dependencies = new ArrayList<String>();
+ service.setDependencies(dependencies);
+ ServiceApiUtil.checkServiceDependencySatisified(service);
+ }
+
+ @Test
+ public void testServiceDependencies() {
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ Service service = createExampleApplication();
+ Component compa = createComponent("compa");
+ Component compb = createComponent("compb");
+ service.addComponent(compa);
+ service.addComponent(compb);
+ List<String> dependencies = new ArrayList<String>();
+ dependencies.add("abc");
+ service.setDependencies(dependencies);
+ Service dependent = createExampleApplication();
+ dependent.setState(ServiceState.STOPPED);
+ ServiceApiUtil.checkServiceDependencySatisified(service);
+ }
+ };
+ thread.start();
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ Assert.assertTrue(thread.isAlive());
+ }
+
public static Service createExampleApplication() {
Service exampleApp = new Service();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index 4bfa742..7b1e74a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -406,7 +406,7 @@ a service resource has the following attributes.
|queue|The YARN queue that this service should be submitted to.|false|string||
|kerberos_principal|The principal info of the user who launches the service|false|KerberosPrincipal||
|docker_client_config|URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json)|false|string||
-
+|dependencies|A list of service names that this service depends on.| false | string array ||
### ServiceState
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org