You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/25 18:03:30 UTC
[48/50] [abbrv] hadoop git commit: YARN-6716. Native services support
for specifying component start order. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java
deleted file mode 100644
index 096838d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * This is the monitor service
- */
-public final class ReportingLoop implements Runnable, ProbeReportHandler, MonitorKeys, Closeable {
- protected static final Logger log = LoggerFactory.getLogger(ReportingLoop.class);
- private final ProbeWorker worker;
- private final Thread workerThread;
- private final int reportInterval;
- private final int probeTimeout;
- private final int bootstrapTimeout;
- private ProbeReportHandler reporter;
- private final String name;
- private volatile boolean mustExit;
-
- public ReportingLoop(String name,
- ProbeReportHandler reporter,
- List<Probe> probes,
- List<Probe> dependencyProbes,
- int probeInterval,
- int reportInterval,
- int probeTimeout,
- int bootstrapTimeout) throws IOException {
- this(name,
- reporter,
- new ProbeWorker(probes, dependencyProbes, probeInterval, bootstrapTimeout),
- reportInterval,
- probeTimeout);
- }
-
- /**
- * Create a new reporting loop -and bond the worker's ProbeReportHandler
- * to us
- * @param name
- * @param reporter
- * @param worker
- * @param reportInterval
- * @param probeTimeout
- */
- public ReportingLoop(String name,
- ProbeReportHandler reporter,
- ProbeWorker worker,
- int reportInterval,
- int probeTimeout) throws IOException {
- this.name = name;
- this.reporter = reporter;
- this.reportInterval = reportInterval;
- this.probeTimeout = probeTimeout;
- this.worker = worker;
- this.bootstrapTimeout = worker.getBootstrapTimeout();
- worker.setReportHandler(this);
- workerThread = new Thread(worker, "probe thread - " + name);
- worker.init();
- }
-
- public int getBootstrapTimeout() {
- return bootstrapTimeout;
- }
-
- public ReportingLoop withReporter(ProbeReportHandler reporter) {
- assert this.reporter == null : "attempting to reassign reporter ";
- assert reporter != null : "new reporter is null";
- this.reporter = reporter;
- return this;
- }
-
- /**
- * Start the monitoring.
- *
- * @return false if the monitoring did not start and that the worker threads
- * should be run up.
- */
- public boolean startReporting() {
- String description = "Service Monitor for " + name + ", probe-interval= "
- + MonitorUtils.millisToHumanTime(worker.interval)
- + ", report-interval=" + MonitorUtils.millisToHumanTime(reportInterval)
- + ", probe-timeout=" + timeoutToStr(probeTimeout)
- + ", bootstrap-timeout=" + timeoutToStr(bootstrapTimeout);
- log.info("Starting reporting"
- + " to " + reporter
- + description);
- return reporter.commence(name, description);
- }
-
- private String timeoutToStr(int timeout) {
- return timeout >= 0 ? MonitorUtils.millisToHumanTime(timeout) : "not set";
- }
-
- private void startWorker() {
- log.info("Starting reporting worker thread ");
- workerThread.setDaemon(true);
- workerThread.start();
- }
-
-
- /**
- * This exits the process cleanly
- */
- @Override
- public void close() {
- log.info("Stopping reporting");
- mustExit = true;
- if (worker != null) {
- worker.setMustExit();
- workerThread.interrupt();
- }
- if (reporter != null) {
- reporter.unregister();
- }
- }
-
- @Override
- public void probeFailure(ProbeFailedException exception) {
- reporter.probeFailure(exception);
- }
-
- @Override
- public void probeProcessStateChange(ProbePhase probePhase) {
- reporter.probeProcessStateChange(probePhase);
- }
-
- @Override
- public void probeBooted(ProbeStatus status) {
- reporter.probeBooted(status);
- }
-
- private long now() {
- return System.currentTimeMillis();
- }
-
- @Override
- public void probeResult(ProbePhase phase, ProbeStatus status) {
- reporter.probeResult(phase, status);
- }
-
- @Override
- public boolean commence(String n, String description) {
- return true;
- }
-
- @Override
- public void unregister() {
- }
-
- @Override
- public void heartbeat(ProbeStatus status) {
- }
-
- @Override
- public void probeTimedOut(ProbePhase currentPhase, Probe probe, ProbeStatus lastStatus,
- long currentTime) {
- }
-
- @Override
- public void liveProbeCycleCompleted() {
- //delegate to the reporter
- reporter.liveProbeCycleCompleted();
- }
-
- /**
- * The reporting loop
- */
- void reportingLoop() {
-
- while (!mustExit) {
- try {
- ProbeStatus workerStatus = worker.getLastStatus();
- long now = now();
- long lastStatusIssued = workerStatus.getTimestamp();
- long timeSinceLastStatusIssued = now - lastStatusIssued;
- //two actions can occur here: a heartbeat is issued or a timeout reported.
- //this flag decides which
- boolean heartbeat;
-
- //based on phase, decide whether to heartbeat or timeout
- ProbePhase probePhase = worker.getProbePhase();
- switch (probePhase) {
- case DEPENDENCY_CHECKING:
- //no timeouts in dependency phase
- heartbeat = true;
- break;
-
- case BOOTSTRAPPING:
- //the timeout here is fairly straightforward: heartbeats are
- //raised while the worker hasn't timed out
- heartbeat = bootstrapTimeout < 0 || timeSinceLastStatusIssued < bootstrapTimeout;
-
- break;
-
- case LIVE:
- //use the probe timeout interval between the current time
- //and the time the last status event was received.
- heartbeat = timeSinceLastStatusIssued < probeTimeout;
- break;
-
- case INIT:
- case TERMINATING:
- default:
- //send a heartbeat, because this isn't the time to be failing
- heartbeat = true;
- }
- if (heartbeat) {
- //a heartbeat is sent to the reporter
- reporter.heartbeat(workerStatus);
- } else {
- //no response from the worker -it is hung.
- reporter.probeTimedOut(probePhase,
- worker.getCurrentProbe(),
- workerStatus,
- now
- );
- }
-
- //now sleep
- Thread.sleep(reportInterval);
-
- } catch (InterruptedException e) {
- //interrupted -always exit the loop.
- break;
- }
- }
- //this point is reached if and only if a clean exit was requested or something failed.
- }
-
- /**
- * This can be run in a separate thread, or it can be run directly from the caller.
- * Test runs do the latter, HAM runs multiple reporting threads.
- */
- @Override
- public void run() {
- try {
- startWorker();
- reportingLoop();
- } catch (RuntimeException e) {
- log.warn("Failure in the reporting loop: " + e, e);
- //rethrow so that inline code can pick it up (e.g. test runs)
- throw e;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
index 676db82..3033537 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
@@ -62,6 +62,10 @@ public interface RestApiErrorMessages {
"Invalid no of containers specified";
String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_DEPENDENCY_INVALID = "Dependency %s for component %s is " +
+ "invalid, does not exist as a component";
+ String ERROR_DEPENDENCY_CYCLE = "Invalid dependencies, a cycle may " +
+ "exist: %s";
String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
"Cannot specify" + " cpus/memory along with profile";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
index 80a31c0..e977727 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
@@ -32,14 +32,18 @@ import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.persist.JsonSerDeser;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.SliderProviderFactory;
+import org.apache.slider.server.servicemonitor.MonitorUtils;
import org.codehaus.jackson.map.PropertyNamingStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
public class ServiceApiUtil {
@@ -176,9 +180,22 @@ public class ServiceApiUtil {
if (comp.getLaunchCommand() == null) {
comp.setLaunchCommand(globalLaunchCommand);
}
+ // validate dependency existence
+ if (comp.getDependencies() != null) {
+ for (String dependency : comp.getDependencies()) {
+ if (!componentNames.contains(dependency)) {
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency,
+ comp.getName()));
+ }
+ }
+ }
validateComponent(comp, fs.getFileSystem());
}
+ // validate dependency tree
+ sortByDependencies(application.getComponents());
+
// Application lifetime if not specified, is set to unlimited lifetime
if (application.getLifetime() == null) {
application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
@@ -207,6 +224,8 @@ public class ServiceApiUtil {
}
compClientProvider.validateConfigFiles(comp.getConfiguration()
.getFiles(), fs);
+
+ MonitorUtils.getProbe(comp.getReadinessCheck());
}
@VisibleForTesting
@@ -301,6 +320,67 @@ public class ServiceApiUtil {
return comp;
}
+ public static Collection<Component> sortByDependencies(List<Component>
+ components) {
+ Map<String, Component> sortedComponents =
+ sortByDependencies(components, null);
+ return sortedComponents.values();
+ }
+
+ /**
+ * Each internal call of sortByDependencies will identify all of the
+ * components with the same dependency depth (the lowest depth that has not
+ * been processed yet) and add them to the sortedComponents list, preserving
+ * their original ordering in the components list.
+ *
+ * So the first time it is called, all components with no dependencies
+ * (depth 0) will be identified. The next time it is called, all components
+ * that have dependencies only on the the depth 0 components will be
+ * identified (depth 1). This will be repeated until all components have
+ * been added to the sortedComponents list. If no new components are
+ * identified but the sortedComponents list is not complete, an error is
+ * thrown.
+ */
+ private static Map<String, Component> sortByDependencies(List<Component>
+ components, Map<String, Component> sortedComponents) {
+ if (sortedComponents == null) {
+ sortedComponents = new LinkedHashMap<>();
+ }
+
+ Map<String, Component> componentsToAdd = new LinkedHashMap<>();
+ List<Component> componentsSkipped = new ArrayList<>();
+ for (Component component : components) {
+ String name = component.getName();
+ if (sortedComponents.containsKey(name)) {
+ continue;
+ }
+ boolean dependenciesAlreadySorted = true;
+ if (!SliderUtils.isEmpty(component.getDependencies())) {
+ for (String dependency : component.getDependencies()) {
+ if (!sortedComponents.containsKey(dependency)) {
+ dependenciesAlreadySorted = false;
+ break;
+ }
+ }
+ }
+ if (dependenciesAlreadySorted) {
+ componentsToAdd.put(name, component);
+ } else {
+ componentsSkipped.add(component);
+ }
+ }
+
+ if (componentsToAdd.size() == 0) {
+ throw new IllegalArgumentException(String.format(RestApiErrorMessages
+ .ERROR_DEPENDENCY_CYCLE, componentsSkipped));
+ }
+ sortedComponents.putAll(componentsToAdd);
+ if (sortedComponents.size() == components.size()) {
+ return sortedComponents;
+ }
+ return sortByDependencies(components, sortedComponents);
+ }
+
public static String $(String s) {
return "${" + s +"}";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
new file mode 100644
index 0000000..2967309
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockRoles;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.state.ContainerAssignment;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.servicemonitor.ProbeStatus;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Test for postponing container requests until dependencies are ready.
+ */
+public class TestMockAppStateDependencies extends BaseMockAppStateTest
+ implements MockRoles {
+
+ private org.apache.slider.server.servicemonitor.Probe successProbe =
+ new org.apache.slider.server.servicemonitor.Probe("success", null) {
+ @Override
+ public ProbeStatus ping(RoleInstance roleInstance) {
+ ProbeStatus status = new ProbeStatus();
+ status.succeed(this);
+ return status;
+ }
+ };
+
+ private org.apache.slider.server.servicemonitor.Probe failureProbe =
+ new org.apache.slider.server.servicemonitor.Probe("failure", null) {
+ @Override
+ public ProbeStatus ping(RoleInstance roleInstance) {
+ ProbeStatus status = new ProbeStatus();
+ status.fail(this, new Exception());
+ return status;
+ }
+ };
+
+ @Override
+ public String getTestName() {
+ return "TestMockAppStateDependencies";
+ }
+
+ @Test
+ public void testDependencies() throws Throwable {
+ RoleStatus role0Status = getRole0Status();
+ RoleStatus role1Status = getRole1Status();
+
+ // set desired instances for role0 to 1
+ role0Status.setDesired(1);
+ // set probe for role0 to use a ping that will always succeed
+ role0Status.getProviderRole().probe = successProbe;
+
+ // set desired instances for role1 to 1
+ role1Status.setDesired(1);
+ // set role0 as a dependency of role1
+ role1Status.getProviderRole().component.setDependencies(Collections
+ .singletonList(ROLE0));
+
+ // role0 has no dependencies, so its dependencies are ready
+ assertTrue(appState.areDependenciesReady(role0Status));
+ // role1 dependency (role0) is not ready yet
+ assertFalse(appState.areDependenciesReady(role1Status));
+ // start the single requested instance for role0
+ review(ROLE0, 2);
+
+ // role0 is still not ready because a ping has not been issued
+ assertFalse(appState.areDependenciesReady(role1Status));
+ // issue pings
+ appState.monitorComponentInstances();
+ // now role0 is ready
+ assertTrue(appState.areDependenciesReady(role1Status));
+ // increase the desired containers for role0
+ role0Status.setDesired(2);
+ // role0 is no longer ready
+ assertFalse(appState.areDependenciesReady(role1Status));
+ // start a second instance for role0
+ review(ROLE0, 2);
+
+ // role0 is not ready because ping has not been issued for the new instance
+ assertFalse(appState.areDependenciesReady(role1Status));
+ // issue pings
+ appState.monitorComponentInstances();
+ // role0 is ready
+ assertTrue(appState.areDependenciesReady(role1Status));
+
+ // set probe for role0 to use a ping that will always fail
+ role0Status.getProviderRole().probe = failureProbe;
+ // issue pings
+ appState.monitorComponentInstances();
+ // role0 is not ready (failure probe works)
+ assertFalse(appState.areDependenciesReady(role1Status));
+ // set probe for role0 to use a ping that will always succeed
+ role0Status.getProviderRole().probe = successProbe;
+ // issue pings
+ appState.monitorComponentInstances();
+ // role0 is ready
+ assertTrue(appState.areDependenciesReady(role1Status));
+
+ // now role1 instances can be started
+ review(ROLE1, 1);
+ }
+
+ public void review(String expectedRole, int outstanding) throws Exception {
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+
+ // expect one request in the list
+ assertEquals(1, ops.size());
+ // and in a liveness check, expected outstanding
+ ApplicationLivenessInformation liveness =
+ appState.getApplicationLivenessInformation();
+ assertEquals(outstanding, liveness.requestsOutstanding);
+ assertFalse(liveness.allRequestsSatisfied);
+
+ // record container allocated and verify it has the expected role
+ List<Container> allocations = engine.execute(ops);
+ List<ContainerAssignment> assignments = new ArrayList<>();
+ List<AbstractRMOperation> releases = new ArrayList<>();
+ appState.onContainersAllocated(allocations, assignments, releases);
+ assertEquals(1, assignments.size());
+ ContainerAssignment assigned = assignments.get(0);
+ Container target = assigned.container;
+ RoleInstance ri = roleInstance(assigned);
+ assertEquals(expectedRole, ri.role);
+
+ // one fewer request outstanding
+ liveness = appState.getApplicationLivenessInformation();
+ assertEquals(outstanding - 1, liveness.requestsOutstanding);
+
+ // record container start submitted
+ appState.containerStartSubmitted(target, ri);
+
+ // additional review results in no additional requests
+ ops = appState.reviewRequestAndReleaseNodes();
+ assertTrue(ops.isEmpty());
+
+ // record container start
+ appState.innerOnNodeManagerContainerStarted(target.getId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
index 703d65f..edc1866 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
@@ -103,8 +103,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assertEquals(i, instance.componentId);
assertEquals(group, instance.role);
assertEquals(group, instance.providerRole.name);
- assertEquals(group, instance.providerRole.group);
- // TODO remove group from provider role if it continues to be unused
i++;
}
}
@@ -124,7 +122,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assertEquals(0, roleStatus.getDesired());
assertEquals(1024L, roleStatus.getResourceRequirements().getMemorySize());
assertEquals(2, roleStatus.getResourceRequirements().getVirtualCores());
- assertEquals("group1", roleStatus.getGroup());
// now flex back up
appState.updateComponents(Collections.singletonMap("group1", 3L));
@@ -147,7 +144,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
RoleStatus group1 = appState.lookupRoleStatus("group1");
assertEquals(3, group1.getDesired());
assertEquals(1024L, group1.getResourceRequirements().getMemorySize());
- assertEquals("group1", group1.getGroup());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
index 7d8f5a7..555db75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
@@ -343,7 +343,7 @@ public class TestRoleHistoryOutstandingRequestTracker extends
public void testBuildResourceRequirements() throws Throwable {
// Store original values
Application application = appState.getClusterStatus();
- Component role0 = application.getComponent(getRole0Status().getGroup());
+ Component role0 = application.getComponent(getRole0Status().getName());
String origMem = role0.getResource().getMemory();
Integer origVcores = role0.getResource().getCpus();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
index 458d1bc..dacfb0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
@@ -17,20 +17,25 @@
package org.apache.slider.server.servicemonitor;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.state.RoleInstance;
import org.junit.Assert;
import org.junit.Test;
public class TestPortProbe extends Assert {
+ private final MockFactory factory = MockFactory.INSTANCE;
+
/**
* Assert that a port probe failed if the port is closed
* @throws Throwable
*/
@Test
public void testPortProbeFailsClosedPort() throws Throwable {
- PortProbe probe = new PortProbe("127.0.0.1", 65500, 100, "", new Configuration());
+ PortProbe probe = new PortProbe(65500, 100);
probe.init();
- ProbeStatus status = probe.ping(true);
+ RoleInstance roleInstance = new RoleInstance(factory.newContainer());
+ roleInstance.ip = "127.0.0.1";
+ ProbeStatus status = probe.ping(roleInstance);
assertFalse("Expected a failure but got successful result: " + status,
status.isSuccess());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
index 9ca3242..d7a9cfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
@@ -34,6 +34,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
import static org.apache.slider.util.RestApiConstants.DEFAULT_COMPONENT_NAME;
import static org.apache.slider.util.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
@@ -390,4 +393,52 @@ public class TestServiceApiUtil {
// original component replaced by external component
assertNotNull(app.getComponent("comp1"));
}
+
+ public static void verifyDependencySorting(List<Component> components,
+ Component... expectedSorting) {
+ Collection<Component> actualSorting = ServiceApiUtil.sortByDependencies(
+ components);
+ assertEquals(expectedSorting.length, actualSorting.size());
+ int i = 0;
+ for (Component component : actualSorting) {
+ assertEquals(expectedSorting[i++], component);
+ }
+ }
+
+ @Test
+ public void testDependencySorting() throws IOException {
+ Component a = new Component().name("a");
+ Component b = new Component().name("b");
+ Component c = new Component().name("c");
+ Component d = new Component().name("d").dependencies(Arrays.asList("c"));
+ Component e = new Component().name("e").dependencies(Arrays.asList("b",
+ "d"));
+
+ verifyDependencySorting(Arrays.asList(a, b, c), a, b, c);
+ verifyDependencySorting(Arrays.asList(c, a, b), c, a, b);
+ verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e);
+ verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e);
+
+ c.setDependencies(Arrays.asList("e"));
+ try {
+ verifyDependencySorting(Arrays.asList(a, b, c, d, e));
+ Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle");
+ } catch (IllegalArgumentException ex) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d,
+ e)), ex.getMessage());
+ }
+
+ SliderFileSystem sfs = initMock(null);
+ Application application = createValidApplication(null);
+ application.setComponents(Arrays.asList(c, d, e));
+ try {
+ ServiceApiUtil.validateAndResolveApplication(application, sfs);
+ Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies");
+ } catch (IllegalArgumentException ex) {
+ assertEquals(String.format(
+ RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex
+ .getMessage());
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org