You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2019/04/15 03:11:30 UTC
[hadoop] branch trunk updated: YARN-9474. Remove hard coded sleep
from Opportunistic Scheduler tests. Contributed by Abhishek Modi.
This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7a68e7a YARN-9474. Remove hard coded sleep from Opportunistic Scheduler tests. Contributed by Abhishek Modi.
7a68e7a is described below
commit 7a68e7abd51b9a0e25758e182eaa0a5cbbf13d87
Author: Inigo Goiri <in...@apache.org>
AuthorDate: Sun Apr 14 20:11:20 2019 -0700
YARN-9474. Remove hard coded sleep from Opportunistic Scheduler tests. Contributed by Abhishek Modi.
---
.../TestOpportunisticContainerAllocator.java | 16 ++++---
...stOpportunisticContainerAllocatorAMService.java | 51 +++++++++-------------
2 files changed, 31 insertions(+), 36 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/scheduler/TestOpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/scheduler/TestOpportunisticContainerAllocator.java
index 2d3b099..6f71b36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/scheduler/TestOpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/scheduler/TestOpportunisticContainerAllocator.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -47,6 +49,8 @@ import java.util.Set;
public class TestOpportunisticContainerAllocator {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestOpportunisticContainerAllocator.class);
private static final int GB = 1024;
private OpportunisticContainerAllocator allocator = null;
private OpportunisticContainerContext oppCntxt = null;
@@ -174,7 +178,7 @@ public class TestOpportunisticContainerAllocator {
List<Container> containers = allocator.allocateContainers(
blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
- System.out.println(containers);
+ LOG.info("Containers: {}", containers);
Set<String> allocatedHosts = new HashSet<>();
for (Container c : containers) {
allocatedHosts.add(c.getNodeHttpAddress());
@@ -242,7 +246,7 @@ public class TestOpportunisticContainerAllocator {
List<Container> containers = allocator.allocateContainers(
blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
- System.out.println(containers);
+ LOG.info("Containers: {}", containers);
Set<String> allocatedHosts = new HashSet<>();
for (Container c : containers) {
allocatedHosts.add(c.getNodeHttpAddress());
@@ -295,7 +299,7 @@ public class TestOpportunisticContainerAllocator {
List<Container> containers = allocator.allocateContainers(
blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
- System.out.println(containers);
+ LOG.info("Containers: {}", containers);
Set<String> allocatedHosts = new HashSet<>();
for (Container c : containers) {
allocatedHosts.add(c.getNodeHttpAddress());
@@ -412,7 +416,7 @@ public class TestOpportunisticContainerAllocator {
for (Container c : containers) {
allocatedHosts.add(c.getNodeHttpAddress());
}
- System.out.println(containers);
+ LOG.info("Containers: {}", containers);
Assert.assertTrue(allocatedHosts.contains("h2:1234"));
Assert.assertTrue(allocatedHosts.contains("h5:1234"));
Assert.assertFalse(allocatedHosts.contains("h3:1234"));
@@ -459,7 +463,7 @@ public class TestOpportunisticContainerAllocator {
for (Container c : containers) {
allocatedHosts.add(c.getNodeHttpAddress());
}
- System.out.println(containers);
+ LOG.info("Containers: {}", containers);
Assert.assertTrue(allocatedHosts.contains("h2:1234"));
Assert.assertTrue(allocatedHosts.contains("h5:1234"));
Assert.assertFalse(allocatedHosts.contains("h3:1234"));
@@ -502,7 +506,7 @@ public class TestOpportunisticContainerAllocator {
List<Container> containers = allocator.allocateContainers(
blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
- System.out.println(containers);
+ LOG.info("Containers: {}", containers);
Assert.assertEquals(2, containers.size());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
index 9a8c02b..bdd4f71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
@@ -94,8 +94,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSc
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo
- .FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -106,17 +105,12 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
-import com.google.common.base.Supplier;
-
-import static org.junit.Assert.fail;
-
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
-import java.util.concurrent.TimeoutException;
/**
* Test cases for {@link OpportunisticContainerAllocatorAMService}.
@@ -223,7 +217,8 @@ public class TestOpportunisticContainerAllocatorAMService {
nm3.nodeHeartbeat(true);
nm4.nodeHeartbeat(true);
- Thread.sleep(1000);
+ GenericTestUtils.waitFor(() ->
+ amservice.getLeastLoadedNodes().size() == 4, 10, 10 * 100);
QueueMetrics metrics = ((CapacityScheduler) scheduler).getRootQueue()
.getMetrics();
@@ -388,7 +383,8 @@ public class TestOpportunisticContainerAllocatorAMService {
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
- Thread.sleep(1000);
+ GenericTestUtils.waitFor(() ->
+ amservice.getLeastLoadedNodes().size() == 2, 10, 10 * 100);
QueueMetrics metrics = ((CapacityScheduler) scheduler).getRootQueue()
.getMetrics();
@@ -513,7 +509,8 @@ public class TestOpportunisticContainerAllocatorAMService {
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
- Thread.sleep(1000);
+ GenericTestUtils.waitFor(() ->
+ amservice.getLeastLoadedNodes().size() == 2, 10, 10 * 100);
QueueMetrics metrics = ((CapacityScheduler) scheduler).getRootQueue()
.getMetrics();
@@ -618,7 +615,9 @@ public class TestOpportunisticContainerAllocatorAMService {
amservice.handle(new NodeUpdateSchedulerEvent(rmNode1));
nm1.nodeHeartbeat(true);
- Thread.sleep(1000);
+
+ GenericTestUtils.waitFor(() ->
+ amservice.getLeastLoadedNodes().size() == 1, 10, 10 * 100);
AllocateResponse allocateResponse = am1.allocate(Arrays.asList(
ResourceRequest.newInstance(Priority.newInstance(1), "*",
@@ -801,6 +800,9 @@ public class TestOpportunisticContainerAllocatorAMService {
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
+ GenericTestUtils.waitFor(() ->
+ amservice.getLeastLoadedNodes().size() == 2, 10, 10 * 100);
+
AllocateResponse allocateResponse = am1.allocate(Arrays.asList(
ResourceRequest.newInstance(Priority.newInstance(1), "*",
Resources.createResource(1 * GB), 2, true, null,
@@ -964,15 +966,10 @@ public class TestOpportunisticContainerAllocatorAMService {
// Send add and update node events to AM Service.
amservice.handle(new NodeAddedSchedulerEvent(rmNode1));
amservice.handle(new NodeUpdateSchedulerEvent(rmNode1));
- try {
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- return scheduler.getNumClusterNodes() == 1;
- }
- }, 10, 200 * 100);
- }catch (TimeoutException e) {
- fail("timed out while waiting for NM to add.");
- }
+
+ GenericTestUtils.waitFor(() ->
+ scheduler.getNumClusterNodes() == 1, 10, 200 * 100);
+
AllocateResponse allocateResponse = am.allocate(
Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
"*", Resources.createResource(1 * GB), 2, true, null,
@@ -983,15 +980,10 @@ public class TestOpportunisticContainerAllocatorAMService {
.getAllocatedContainers();
Container container = allocatedContainers.get(0);
scheduler.handle(new NodeRemovedSchedulerEvent(rmNode1));
- try {
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- return scheduler.getNumClusterNodes() == 0;
- }
- }, 10, 200 * 100);
- }catch (TimeoutException e) {
- fail("timed out while waiting for NM to remove.");
- }
+
+ GenericTestUtils.waitFor(() ->
+ scheduler.getNumClusterNodes() == 0, 10, 200 * 100);
+
//test YARN-9165
RMContainer rmContainer = null;
rmContainer = SchedulerUtils.createOpportunisticRmContainer(
@@ -1002,7 +994,6 @@ public class TestOpportunisticContainerAllocatorAMService {
schedulerAttempt.getApplicationAttemptId(), container.getNodeId(),
schedulerAttempt.getUser(), rm.getRMContext(), true);
}
- assert(rmContainer!=null);
//test YARN-9164
schedulerAttempt.addRMContainer(container.getId(), rmContainer);
scheduler.handle(new AppAttemptRemovedSchedulerEvent(attemptId,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org