You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@myriad.apache.org by sm...@apache.org on 2015/10/28 17:07:41 UTC
[09/20] incubator-myriad git commit: com.ebay => org.apache
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java
deleted file mode 100644
index 3567c3f..0000000
--- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.ebay.myriad.scheduler;
-
-import com.ebay.myriad.scheduler.yarn.MyriadFairScheduler;
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-//import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-//import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-//import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
-//import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
-//import org.apache.hadoop.yarn.util.resource.Resources;
-import org.junit.After;
-import org.junit.Before;
-//import org.junit.Test;
-
-//import static org.junit.Assert.assertEquals;
-
-/**
- * Tests myriad scheduler.
- */
-public class TestMyriadScheduler {
- protected Configuration conf;
- protected FairScheduler scheduler;
- protected ResourceManager resourceManager;
-
- @Before
- public void setUp() throws IOException {
- scheduler = new MyriadFairScheduler();
- conf = createConfiguration();
- resourceManager = new ResourceManager();
- resourceManager.init(conf);
-
- // TODO: This test should really be using MockRM. For now starting stuff
- // that is needed at a bare minimum.
- ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
- resourceManager.getRMContext().getStateStore().start();
-
- // to initialize the master key
- resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
-
- scheduler.setRMContext(resourceManager.getRMContext());
- scheduler.init(conf);
- scheduler.start();
- }
-
- @After
- public void tearDown() {
- if (scheduler != null) {
- scheduler.stop();
- scheduler = null;
- }
-
- if (resourceManager != null) {
- resourceManager.stop();
- resourceManager = null;
- }
- }
-
- public Configuration createConfiguration() {
- Configuration conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class);
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
- conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1024);
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240);
- return conf;
- }
-/*
- @Test
- public void testClusterMemory() throws Exception {
- // Add a node
- RMNode node1 =
- MockNodes
- .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
- scheduler.handle(nodeEvent1);
- assertEquals(1024, scheduler.getClusterResource().getMemory());
-
- // Add another node
- RMNode node2 =
- MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2");
- NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
- scheduler.handle(nodeEvent2);
- assertEquals(1536, scheduler.getClusterResource().getMemory());
-
- // Remove the first node
- NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1);
- scheduler.handle(nodeEvent3);
- assertEquals(512, scheduler.getClusterResource().getMemory());
- }
- */
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java
deleted file mode 100644
index f785f2e..0000000
--- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestServiceCommandLine.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.ebay.myriad.scheduler;
-
-import static org.junit.Assert.*;
-
-import org.apache.mesos.Protos.CommandInfo;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.ebay.myriad.configuration.MyriadConfiguration;
-import com.ebay.myriad.scheduler.TaskFactory.NMTaskFactoryImpl;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-
-/**
- * Class to test CommandLine generation
- */
-public class TestServiceCommandLine {
-
- static MyriadConfiguration cfg;
-
- static String toJHSCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . &&" +
- " cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\";" +
- "sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver";
-
- static String toCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;";
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
- cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class);
-
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- }
-
- @Test
- public void testJHSCommandLineGeneration() throws Exception {
- ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null);
- String executorCmd = "$YARN_HOME/bin/mapred historyserver";
- ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0);
-
- CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd);
-
- assertTrue(cInfo.getValue().startsWith(toCompare));
- }
-
- @Test
- public void testNMCommandLineGeneration() throws Exception {
- Long[] ports = new Long[]{1L, 2L, 3L, 4L};
- NMPorts nmPorts = new NMPorts(ports);
-
- ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L, 15L), 3.0, 5.0);
-
- ExecutorCommandLineGenerator clGenerator = new DownloadNMExecutorCLGenImpl(cfg, "hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz");
- NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator);
-
- CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts);
-
- assertTrue(cInfo.getValue().startsWith(toCompare));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java
deleted file mode 100644
index f296f63..0000000
--- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestTaskUtils.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.ebay.myriad.scheduler;
-
-import static org.junit.Assert.*;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.ebay.myriad.configuration.MyriadBadConfigurationException;
-import com.ebay.myriad.configuration.MyriadConfiguration;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-/**
- * Tests for TaskUtils
- */
-public class TestTaskUtils {
-
- static MyriadConfiguration cfg;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
- cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class);
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- }
-
- @Test
- public void testGetResource() {
- TaskUtils taskUtils = new TaskUtils(cfg);
-
- NMProfile fooProfile = new NMProfile("abc", 1L, 1000L);
- try {
- taskUtils.getAuxTaskCpus(fooProfile, "foo");
- fail("Should not complete sucessfully for foo");
- } catch (MyriadBadConfigurationException e) {
- // success
- }
-
- try {
- double cpu = taskUtils.getAuxTaskCpus(fooProfile, "jobhistory");
- assertTrue(cpu > 0.0);
- } catch (MyriadBadConfigurationException e) {
- fail("cpu should be defined for jobhistory");
- }
- }
-
- @Test
- public void testServiceResourceProfile() throws Exception {
- // testing custom deserializer
-
- Gson gson = new GsonBuilder().registerTypeAdapter(ServiceResourceProfile.class, new ServiceResourceProfile.CustomDeserializer()).create();
-
-
- ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0);
-
- String parentStr = gson.toJson(parentProfile);
- ServiceResourceProfile processedProfile = gson.fromJson(parentStr, ServiceResourceProfile.class);
-
- assertTrue(processedProfile.getClass().equals(ServiceResourceProfile.class));
- assertTrue(processedProfile.toString().equalsIgnoreCase(parentStr));
-
- ServiceResourceProfile childProfile = new ExtendedResourceProfile(new NMProfile("bcd", 5L, 15L), 2.0, 7.0);
-
- String childStr = gson.toJson(childProfile);
- ServiceResourceProfile processedChildProfile = gson.fromJson(childStr, ServiceResourceProfile.class);
-
- assertTrue(processedChildProfile instanceof ExtendedResourceProfile);
- assertTrue(processedChildProfile.toString().equalsIgnoreCase(childStr));
- }
-
- @Test
-
- public void testStackTrace() {
-
- new Throwable().printStackTrace();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/constraints/LikeConstraintSpec.groovy
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/constraints/LikeConstraintSpec.groovy b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/constraints/LikeConstraintSpec.groovy
deleted file mode 100644
index 497564e..0000000
--- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/constraints/LikeConstraintSpec.groovy
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.ebay.myriad.scheduler.constraints
-
-import com.google.common.collect.Lists
-import org.apache.mesos.Protos
-import spock.lang.Specification
-
-import static org.apache.mesos.Protos.Value.Text
-import static org.apache.mesos.Protos.Value.Type.TEXT
-
-/**
- *
- * Test for LikeConstraint
- *
- */
-class LikeConstraintSpec extends Specification {
-
- def "is matching host name"() {
- given:
- def constraint = new LikeConstraint("hostname", "host-[0-9]*.example.com")
-
- expect:
- returnValue == constraint.matchesHostName(inputHostName)
-
- where:
- inputHostName | returnValue
- null | false
- "" | false
- "blah-blue" | false
- "host-12.example.com" | true
- "host-1.example.com" | true
- "host-2.example.com" | true
- }
-
- def "is matching dfs attribute"() {
- given:
- def constraint = new LikeConstraint("dfs", "true")
-
- expect:
- returnValue == constraint.matchesSlaveAttributes(attributes)
-
- where:
- attributes | returnValue
- null | false
- Lists.newArrayList() | false
- Lists.newArrayList(getTextAttribute("dfs", "")) | false
- Lists.newArrayList(getTextAttribute("dfs", "false")) | false
- Lists.newArrayList(getTextAttribute("Distributed FS", "true")) | false
- Lists.newArrayList(getTextAttribute("dfs", "true")) | true
- Lists.newArrayList(getTextAttribute("dfs", "true"),
- getTextAttribute("random", "random value")) | true
- }
-
- def "equals"() {
- given:
- def constraint1 = new LikeConstraint("hostname", "perfnode13[3-4].perf.lab")
- def constraint2 = new LikeConstraint("hostname", "perfnode13[3-4].perf.lab")
- def constraint3 = new LikeConstraint("hostname", "perfnode133.perf.lab")
-
- expect:
- constraint1.equals(constraint2)
- !constraint1.equals(constraint3)
- !constraint2.equals(constraint3)
- }
-
- private static Protos.Attribute getTextAttribute(String name, String value) {
- Protos.Attribute.newBuilder()
- .setName(name)
- .setType(TEXT)
- .setText(Text.newBuilder()
- .setValue(value))
- .build()
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/FGSTestBaseSpec.groovy b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
deleted file mode 100644
index 166d0f6..0000000
--- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.ebay.myriad.scheduler.fgs
-
-import com.ebay.myriad.configuration.MyriadConfiguration
-import com.ebay.myriad.scheduler.MyriadDriver
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
-import org.apache.hadoop.yarn.api.records.*
-import org.apache.hadoop.yarn.event.Dispatcher
-import org.apache.hadoop.yarn.event.EventHandler
-import org.apache.hadoop.yarn.server.resourcemanager.MockNodes
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext
-import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter
-import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode
-import org.apache.hadoop.yarn.util.resource.Resources
-import org.apache.mesos.Protos
-import org.apache.mesos.SchedulerDriver
-import spock.lang.Specification
-
-import java.util.concurrent.ConcurrentHashMap
-
-/**
- *
- * Base class for testing Fine Grained Scaling.
- *
- */
-class FGSTestBaseSpec extends Specification {
- def nodeStore = new NodeStore()
- def mesosDriver = Mock(SchedulerDriver)
- def myriadDriver = new MyriadDriver(mesosDriver)
- def offerLifecycleManager = new OfferLifecycleManager(nodeStore, myriadDriver)
-
- def cfg = new MyriadConfiguration()
-
- void setup() {
- ObjectMapper mapper = new ObjectMapper(new YAMLFactory())
- cfg = mapper.readValue(
- Thread.currentThread().getContextClassLoader().getResource("myriad-config-default.yml"),
- MyriadConfiguration.class)
- }
-/******************* Nodes Related ****************/
-
- def rmNodes = new ConcurrentHashMap<NodeId, RMNode>()
-
- RMNode getRMNode(int cpu, int mem, String host, Protos.SlaveID slaveId) {
- RMNode rmNode = MockNodes.newNodeInfo(0, Resources.createResource(mem, cpu), 0, host)
- if (rmNodes[rmNode.getNodeID()]) {
- throw new IllegalArgumentException("Node with hostname: " + host + " already exists")
- }
- rmNodes.put(rmNode.getNodeID(), rmNode)
- nodeStore.add(getSchedulerNode(rmNode))
- def node = nodeStore.getNode(host)
- node.setSlaveId(slaveId)
-
- return rmNode
- }
-
- SchedulerNode getSchedulerNode(RMNode rmNode) {
- SchedulerNode schedulerNode = new SchedulerNode(rmNode, false) {
-
- @Override
- void reserveResource(SchedulerApplicationAttempt attempt, Priority priority, RMContainer container) {
- }
-
- @Override
- void unreserveResource(SchedulerApplicationAttempt attempt) {
- }
- }
- return schedulerNode
- }
-
- /******************* RMContext Related ****************/
-
- def publisher = Mock(SystemMetricsPublisher) {}
- def writer = Mock(RMApplicationHistoryWriter) {}
- def handler = Mock(EventHandler) {}
-
- def dispatcher = Mock(Dispatcher) {
- getEventHandler() >> handler
- }
-
- def rmContext = Mock(RMContext) {
- getDispatcher() >> dispatcher
- getRMApplicationHistoryWriter() >> writer
- getSystemMetricsPublisher() >> publisher
- getRMNodes() >> rmNodes
- }
-
- /******************* Offers Related ****************/
-
- Protos.Offer addOfferToFeed(Protos.SlaveID slaveID, String host, int cpu, int mem) {
- def offer = Protos.Offer.newBuilder()
- .setId(Protos.OfferID.newBuilder().setValue("test_offer_id"))
- .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("test_framework_id"))
- .setSlaveId(slaveID)
- .setHostname(host)
- .addResources(Protos.Resource.newBuilder()
- .setName("cpus")
- .setScalar(Protos.Value.Scalar.newBuilder().setValue(cpu))
- .setType(Protos.Value.Type.SCALAR).build())
- .addResources(Protos.Resource.newBuilder()
- .setName("mem")
- .setScalar(Protos.Value.Scalar.newBuilder().setValue(mem))
- .setType(Protos.Value.Type.SCALAR).build())
- .build()
- offerLifecycleManager.addOffers(offer)
- return offer
- }
-
- /******************* Containers Related ****************/
-
- class FGSContainer {
- ContainerId containerId
- Container container
- RMContainer rmContainer
- ContainerStatus containerStatus
- }
-
- def fgsContainers = new HashMap<>()
-
- AbstractYarnScheduler yarnScheduler = Mock(AbstractYarnScheduler) {
- getRMContainer(_ as ContainerId) >> { ContainerId cid -> fgsContainers.get(cid).rmContainer }
- }
-
- FGSContainer getFGSContainer(RMNode node, int cid, int cpu, int mem, ContainerState state) {
- FGSContainer fgsContainer = createFGSContainer(node, cid, cpu, mem, state)
- if (!fgsContainers[fgsContainer.containerId]) {
- fgsContainers[fgsContainer.containerId] = fgsContainer
- }
- return fgsContainer
- }
-
- private FGSContainer createFGSContainer(RMNode node, int cid, int cpu, int mem, ContainerState state) {
- ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(
- ApplicationId.newInstance(123456789, 1), 1), cid)
- FGSContainer fgsContainer = new FGSContainer()
- fgsContainer.containerId = containerId
- fgsContainer.container = Container.newInstance(containerId, node.getNodeID(), node.getHttpAddress(),
- Resources.createResource(mem, cpu), null, null)
- fgsContainer.rmContainer = new RMContainerImpl(fgsContainer.container, containerId.getApplicationAttemptId(),
- node.getNodeID(), "user1", rmContext)
- nodeStore.getNode(node.getNodeID().getHost()).getNode().allocateContainer(fgsContainer.rmContainer)
- fgsContainer.containerStatus = ContainerStatus.newInstance(containerId, state, "", 0)
- return fgsContainer
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/NMHeartBeatHandlerSpec.groovy
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/NMHeartBeatHandlerSpec.groovy b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/NMHeartBeatHandlerSpec.groovy
deleted file mode 100644
index a37b9bf..0000000
--- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/NMHeartBeatHandlerSpec.groovy
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.ebay.myriad.scheduler.fgs
-
-import com.ebay.myriad.scheduler.yarn.interceptor.InterceptorRegistry
-import com.ebay.myriad.state.SchedulerState
-import org.apache.hadoop.yarn.api.records.ContainerState
-import org.apache.hadoop.yarn.api.records.ContainerStatus
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStartedEvent
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent
-import org.apache.hadoop.yarn.util.resource.Resources
-import org.apache.mesos.Protos
-import org.slf4j.Logger
-
-/**
- *
- * Tests for NMHeartBeatHandler
- *
- */
-class NMHeartBeatHandlerSpec extends FGSTestBaseSpec {
-
- def "Node Manager registration"() {
- given:
- def hbHandler = getNMHeartBeatHandler()
- hbHandler.logger = Mock(Logger)
-
- def nonZeroNM = getRMNode(2, 2048, "test_host1", null)
- def zeroNM = getRMNode(0, 0, "test_host2", null)
-
- when:
- hbHandler.beforeRMNodeEventHandled(getNMRegistrationEvent(nonZeroNM), rmContext)
-
- then:
- 1 * hbHandler.logger.warn('FineGrainedScaling feature got invoked for a NM with non-zero capacity. ' +
- 'Host: {}, Mem: {}, CPU: {}. Setting the NM\'s capacity to (0G,0CPU)', 'test_host1', 2048, 2)
- nonZeroNM.getTotalCapability().getMemory() == 0
- nonZeroNM.getTotalCapability().getVirtualCores() == 0
-
- when:
- hbHandler.beforeRMNodeEventHandled(getNMRegistrationEvent(zeroNM), rmContext)
-
- then:
- 0 * hbHandler.logger.warn(_) // no logger.warn invoked
- nonZeroNM.getTotalCapability().getMemory() == 0
- nonZeroNM.getTotalCapability().getVirtualCores() == 0
- }
-
- def "Node Manager HeartBeat"() {
- given:
- def host = "test_host"
- def slaveId = Protos.SlaveID.newBuilder().setValue(host + "_slave_id").build()
- def zeroNM = getRMNode(0, 0, host, slaveId)
-
- def fgsContainer1 = getFGSContainer(zeroNM, 1, 1, 1024, ContainerState.RUNNING)
- def fgsContainer2 = getFGSContainer(zeroNM, 2, 1, 1024, ContainerState.COMPLETE)
- def fgsContainer3 = getFGSContainer(zeroNM, 3, 1, 1024, ContainerState.RUNNING)
-
- addOfferToFeed(slaveId, host, 2, 2048)
-
- def yarnNodeCapacityManager = Mock(YarnNodeCapacityManager)
- def hbHandler = getNMHeartBeatHandler(yarnNodeCapacityManager)
-
- when:
- hbHandler.handleStatusUpdate(
- getHBEvent(
- zeroNM,
- fgsContainer1.containerStatus,
- fgsContainer2.containerStatus,
- fgsContainer3.containerStatus),
- rmContext)
-
- then:
- nodeStore.getNode(host).getContainerSnapshot().size() == 3
- 1 * yarnNodeCapacityManager.setNodeCapacity(zeroNM, Resources.createResource(4096, 4))
- }
-
-
- RMNodeStartedEvent getNMRegistrationEvent(RMNode node) {
- new RMNodeStartedEvent(node.getNodeID(), null, null)
- }
-
- RMNodeStatusEvent getHBEvent(RMNode node, ContainerStatus... statuses) {
- return new RMNodeStatusEvent(node.getNodeID(), null, Arrays.asList(statuses), null, null)
- }
-
- NMHeartBeatHandler getNMHeartBeatHandler() {
- return getNMHeartBeatHandler(Mock(YarnNodeCapacityManager))
- }
-
- NMHeartBeatHandler getNMHeartBeatHandler(YarnNodeCapacityManager yarnNodeCapacityMgr) {
- def registry = Mock(InterceptorRegistry)
- def state = Mock(SchedulerState)
- return new NMHeartBeatHandler(registry, yarnScheduler, myriadDriver,
- yarnNodeCapacityMgr, offerLifecycleManager, nodeStore, state)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/YarnNodeCapacityManagerSpec.groovy
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/YarnNodeCapacityManagerSpec.groovy b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/YarnNodeCapacityManagerSpec.groovy
deleted file mode 100644
index c63ff9a..0000000
--- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/fgs/YarnNodeCapacityManagerSpec.groovy
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.ebay.myriad.scheduler.fgs
-
-import com.ebay.myriad.configuration.NodeManagerConfiguration
-import com.ebay.myriad.scheduler.TaskFactory
-import com.ebay.myriad.scheduler.TaskUtils
-import com.ebay.myriad.scheduler.yarn.interceptor.InterceptorRegistry
-import com.ebay.myriad.state.NodeTask
-import com.ebay.myriad.state.SchedulerState
-import org.apache.hadoop.yarn.api.records.ContainerState
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent
-import org.apache.hadoop.yarn.util.resource.Resources
-import org.apache.mesos.Protos
-
-/**
- *
- * Tests for YarnNodeCapacityManager
- *
- */
-class YarnNodeCapacityManagerSpec extends FGSTestBaseSpec {
-
- def "No Containers Allocated Due To Mesos Offers"() {
- given:
- def yarnNodeCapacityMgr = getYarnNodeCapacityManager()
-
- def host = "test_host"
- def slaveId = Protos.SlaveID.newBuilder().setValue(host + "_slave_id").build()
- def zeroNM = getRMNode(0, 0, host, slaveId)
-
- // have a mesos offer before HB
- def offer = addOfferToFeed(slaveId, host, 4, 4096)
- offerLifecycleManager.markAsConsumed(offer)
-
- // 2 containers before HB.
- def fgsContainer1 = getFGSContainer(zeroNM, 1, 1, 1024, ContainerState.RUNNING)
- def fgsContainer2 = getFGSContainer(zeroNM, 2, 1, 1024, ContainerState.RUNNING)
- nodeStore.getNode(host).snapshotRunningContainers()
-
- // Node's capacity set to match the size of 2 containers + mesos offers
- yarnNodeCapacityMgr.setNodeCapacity(zeroNM, Resources.createResource(6144, 6))
-
- // no new container allocations
-
- when:
- yarnNodeCapacityMgr.handleContainerAllocation(zeroNM)
-
- then:
- nodeStore.getNode(host).getNode().getRunningContainers().size() == 2 // 2 containers still running
- 1 * mesosDriver.declineOffer(offer.getId()) // offer rejected, as it's not used to allocate more containers
- zeroNM.getTotalCapability().getVirtualCores() == 2 // capacity returns back to match size of running containers
- zeroNM.getTotalCapability().getMemory() == 2048
- nodeStore.getNode(host).getContainerSnapshot() == null // container snapshot is released
- }
-
- def "Containers Allocated Due To Mesos Offers"() {
- given:
- def yarnNodeCapacityMgr = getYarnNodeCapacityManager()
-
- def host = "test_host"
- def slaveId = Protos.SlaveID.newBuilder().setValue(host + "_slave_id").build()
- def zeroNM = getRMNode(0, 0, host, slaveId)
-
- // have a mesos offer before HB
- def offer = addOfferToFeed(slaveId, host, 4, 4096)
- offerLifecycleManager.markAsConsumed(offer)
-
- // 2 containers before HB.
- def fgsContainer1 = getFGSContainer(zeroNM, 1, 1, 1024, ContainerState.RUNNING)
- def fgsContainer2 = getFGSContainer(zeroNM, 2, 1, 1024, ContainerState.RUNNING)
- nodeStore.getNode(host).snapshotRunningContainers()
-
- // Node's capacity set to match the size of 2 running containers + mesos offers
- yarnNodeCapacityMgr.setNodeCapacity(zeroNM, Resources.createResource(6144, 6))
-
- // 2 new containers allocated after HB
- def fgsContainer3 = getFGSContainer(zeroNM, 3, 1, 1024, ContainerState.NEW)
- def fgsContainer4 = getFGSContainer(zeroNM, 4, 1, 1024, ContainerState.NEW)
-
- when:
- yarnNodeCapacityMgr.handleContainerAllocation(zeroNM)
-
- then:
- nodeStore.getNode(host).getNode().getRunningContainers().size() == 4 // 2 running + 2 new
- 1 * mesosDriver.launchTasks(_ as Collection<Protos.OfferID>, _ as List<Protos.TaskInfo>) // for place holder tasks
- zeroNM.getTotalCapability().getVirtualCores() == 4 // capacity equals size of running + new containers
- zeroNM.getTotalCapability().getMemory() == 4096
- nodeStore.getNode(host).getContainerSnapshot() == null // container snapshot is released
- }
-
- def "Set Node Capacity"() {
- given:
- def zeroNM = getRMNode(0, 0, "test_host", null)
- def yarnNodeCapacityMgr = getYarnNodeCapacityManager()
-
- when:
- yarnNodeCapacityMgr.setNodeCapacity(zeroNM, Resources.createResource(2048, 2))
-
- then:
- zeroNM.getTotalCapability().getMemory() == 2048
- zeroNM.getTotalCapability().getVirtualCores() == 2
- 1 * rmContext.getDispatcher().getEventHandler().handle(_ as NodeResourceUpdateSchedulerEvent)
- }
-
- YarnNodeCapacityManager getYarnNodeCapacityManager() {
- def registry = Mock(InterceptorRegistry)
- def executorInfo = Protos.ExecutorInfo.newBuilder()
- .setExecutorId(Protos.ExecutorID.newBuilder().setValue("some_id"))
- .setCommand(Protos.CommandInfo.newBuilder())
- .build()
- def nodeTask = Mock(NodeTask) {
- getExecutorInfo() >> executorInfo
- }
- def state = Mock(SchedulerState) {
- getNodeTask(_, NodeManagerConfiguration.NM_TASK_PREFIX) >> nodeTask
- }
- return new YarnNodeCapacityManager(registry, yarnScheduler, rmContext,
- myriadDriver, offerLifecycleManager, nodeStore, state)
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/MesosModule.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/MesosModule.java b/myriad-scheduler/src/test/java/org/apache/myriad/MesosModule.java
new file mode 100644
index 0000000..aee04a8
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/MesosModule.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad;
+
+import java.util.concurrent.FutureTask;
+
+import org.apache.mesos.Protos;
+import org.apache.mesos.Protos.Status;
+import org.apache.mesos.SchedulerDriver;
+import org.apache.mesos.state.State;
+import org.apache.mesos.state.Variable;
+import org.mockito.Mockito;
+
+import org.apache.myriad.configuration.MyriadConfiguration;
+import org.apache.myriad.state.MyriadState;
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import com.google.inject.Scopes;
+import com.google.inject.Singleton;
+
+/**
+ * Guice Module for Mesos objects.
+ */
+public class MesosModule extends AbstractModule {
+ public MesosModule() {
+ }
+
+ @Override
+ protected void configure() {
+ bind(org.apache.myriad.scheduler.MyriadDriver.class).in(Scopes.SINGLETON);
+ }
+
+ @Provides
+ @Singleton
+ SchedulerDriver providesSchedulerDriver(org.apache.myriad.scheduler.MyriadScheduler scheduler, MyriadConfiguration cfg, org.apache.myriad.state.SchedulerState schedulerState) {
+
+ SchedulerDriver driver = Mockito.mock(SchedulerDriver.class);
+ Mockito.when(driver.start()).thenReturn(Status.DRIVER_RUNNING);
+ Mockito.when(driver.abort()).thenReturn(Status.DRIVER_ABORTED);
+
+ return driver;
+ }
+
+ @Provides
+ @Singleton
+ State providesStateStore(MyriadConfiguration cfg) {
+ State stateStore = Mockito.mock(State.class);
+
+ Runnable dummyTask = new Runnable() {
+ public void run() {
+ }
+ };
+
+ Variable var = Mockito.mock(Variable.class);
+ Protos.FrameworkID id = Protos.FrameworkID.newBuilder().setValue("1").build();
+
+ Mockito.when(var.value()).thenReturn(id.toByteArray());
+ FutureTask<Variable> futureTask = new FutureTask<Variable>(dummyTask, var);
+ futureTask.run();
+ Mockito.when(stateStore.fetch(MyriadState.KEY_FRAMEWORK_ID)).thenReturn(futureTask);
+
+ return stateStore;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java
new file mode 100644
index 0000000..66ddb15
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad;
+
+import static org.junit.Assert.*;
+
+import java.util.Map;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.myriad.scheduler.TaskFactory;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+/**
+ * Test for Multibindings
+ */
+public class MultiBindingsTest {
+
+ private static Injector injector;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ MyriadTestModule myriadModule = new MyriadTestModule();
+ injector = Guice.createInjector(myriadModule);
+
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ }
+
+ @Test
+ public void multiBindingsTest() {
+
+
+ MultiBindingsUsage myinstance = injector.getInstance(MultiBindingsUsage.class);
+
+ Map<String, TaskFactory> taskMap = myinstance.getMap();
+ assertNotNull(taskMap);
+ assertEquals(3, taskMap.size());
+
+ taskMap = myinstance.getMap();
+ for (Map.Entry<String, TaskFactory> entry : taskMap.entrySet()) {
+ String keyName = entry.getKey();
+ TaskFactory taskFactory = entry.getValue();
+ System.out.println(taskFactory);
+ }
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsUsage.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsUsage.java b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsUsage.java
new file mode 100644
index 0000000..ecb7894
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsUsage.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad;
+
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import org.apache.myriad.scheduler.TaskFactory;
+
+/**
+ * Helper class to test multibindings
+ */
+public class MultiBindingsUsage {
+
+ @Inject
+ private Map<String, TaskFactory> taskFactoryMap;
+
+ public Map<String, TaskFactory> getMap() {
+ return taskFactoryMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/MyriadTestModule.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/MyriadTestModule.java b/myriad-scheduler/src/test/java/org/apache/myriad/MyriadTestModule.java
new file mode 100644
index 0000000..315d055
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/MyriadTestModule.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.myriad.configuration.ServiceConfiguration;
+import org.apache.myriad.configuration.MyriadConfiguration;
+import org.apache.myriad.configuration.MyriadExecutorConfiguration;
+import org.apache.myriad.configuration.NodeManagerConfiguration;
+import org.apache.myriad.scheduler.TaskFactory.NMTaskFactoryImpl;
+import org.apache.myriad.scheduler.NMExecutorCLGenImpl;
+import org.apache.myriad.scheduler.TaskFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import com.google.inject.Scopes;
+import com.google.inject.Singleton;
+import com.google.inject.multibindings.MapBinder;
+
+/**
+ * AbstractModule extension for UnitTests
+ */
+public class MyriadTestModule extends AbstractModule {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(MyriadTestModule.class);
+
+ private MyriadConfiguration cfg;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void configure() {
+
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ try {
+ cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class);
+ } catch (IOException e1) {
+ LOGGER.error("IOException", e1);
+ return;
+ }
+
+ if (cfg == null) {
+ return;
+ }
+
+ bind(MyriadConfiguration.class).toInstance(cfg);
+
+ MapBinder<String, TaskFactory> mapBinder = MapBinder.newMapBinder(binder(), String.class, TaskFactory.class);
+ mapBinder.addBinding(NodeManagerConfiguration.NM_TASK_PREFIX).to(NMTaskFactoryImpl.class).in(Scopes.SINGLETON);
+ Map<String, ServiceConfiguration> auxServicesConfigs = cfg.getServiceConfigurations();
+ for (Map.Entry<String, ServiceConfiguration> entry : auxServicesConfigs.entrySet()) {
+ String taskFactoryClass = entry.getValue().getTaskFactoryImplName().orNull();
+ if (taskFactoryClass != null) {
+ try {
+ Class<? extends TaskFactory> implClass = (Class<? extends TaskFactory>) Class.forName(taskFactoryClass);
+ mapBinder.addBinding(entry.getKey()).to(implClass).in(Scopes.SINGLETON);
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ }
+ } else {
+ mapBinder.addBinding(entry.getKey()).to(org.apache.myriad.scheduler.ServiceTaskFactoryImpl.class).in(Scopes.SINGLETON);
+ }
+ }
+ }
+
+ @Provides
+ @Singleton
+ org.apache.myriad.scheduler.ExecutorCommandLineGenerator providesCLIGenerator(MyriadConfiguration cfg) {
+ org.apache.myriad.scheduler.ExecutorCommandLineGenerator cliGenerator = null;
+ MyriadExecutorConfiguration myriadExecutorConfiguration = cfg.getMyriadExecutorConfiguration();
+ if (myriadExecutorConfiguration.getNodeManagerUri().isPresent()) {
+ cliGenerator = new org.apache.myriad.scheduler.DownloadNMExecutorCLGenImpl(cfg, myriadExecutorConfiguration.getNodeManagerUri().get());
+ } else {
+ cliGenerator = new NMExecutorCLGenImpl(cfg);
+ }
+ return cliGenerator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadBadConfigurationExceptionTest.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadBadConfigurationExceptionTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadBadConfigurationExceptionTest.java
new file mode 100644
index 0000000..cc4124c
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadBadConfigurationExceptionTest.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad.configuration;
+
+import static org.junit.Assert.*;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Class to test MyriadBadConfigurationException
+ */
+public class MyriadBadConfigurationExceptionTest {
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ }
+
+ @Test
+ public void myriadExceptionTest() {
+ final String testStr = "org.apache.myriad.configuration.MyriadBadConfigurationException: Bad configuration exception";
+ MyriadBadConfigurationException exp = new MyriadBadConfigurationException("Bad configuration exception");
+
+ assertEquals(testStr, exp.toString());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java
new file mode 100644
index 0000000..06f44b6
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad.configuration;
+
+import static org.junit.Assert.*;
+
+import java.util.Map;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ * AuxServices/tasks test
+ */
+public class MyriadConfigurationTest {
+
+ static MyriadConfiguration cfg;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class);
+
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ }
+
+ @Test
+ public void additionalPropertiestest() throws Exception {
+
+ Map<String, ServiceConfiguration> auxConfigs = cfg.getServiceConfigurations();
+
+ assertNotNull(auxConfigs);
+ assertEquals(auxConfigs.size(), 2);
+
+ for (Map.Entry<String, ServiceConfiguration> entry : auxConfigs.entrySet()) {
+ String taskName = entry.getKey();
+ ServiceConfiguration config = entry.getValue();
+ String outTaskname = config.getTaskName();
+ assertEquals(taskName, outTaskname);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsSpec.groovy
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsSpec.groovy b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsSpec.groovy
new file mode 100644
index 0000000..190db32
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsSpec.groovy
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad.scheduler
+
+import org.apache.myriad.configuration.NodeManagerConfiguration
+import org.apache.myriad.state.NodeTask
+import org.apache.myriad.state.SchedulerState
+import org.apache.mesos.Protos
+import spock.lang.Specification
+
+/**
+ *
+ * @author kensipe
+ */
+class SchedulerUtilsSpec extends Specification {
+
+ def "is unique host name"() {
+ given:
+ def offer = Mock(Protos.OfferOrBuilder)
+ offer.getHostname() >> "hostname"
+
+ expect:
+ returnValue == SchedulerUtils.isUniqueHostname(offer, launchTask, tasks)
+
+ where:
+ tasks | launchTask | returnValue
+ [] | null | true
+ null | null | true
+ createNodeTaskList("hostname") | createNodeTask("hostname") | false
+ createNodeTaskList("missinghost") | createNodeTask("hostname") | true
+ createNodeTaskList("missinghost1", "missinghost2") | createNodeTask("missinghost3") | true
+ createNodeTaskList("missinghost1", "hostname") | createNodeTask("hostname") | false
+
+ }
+
+ def "is eligible for Fine Grained Scaling"() {
+ given:
+ def state = Mock(SchedulerState)
+ def tasks = []
+ def fgsNMTask = new NodeTask(new ExtendedResourceProfile(new NMProfile("zero", 0, 0), 1.0, 2.0), null)
+ def cgsNMTask = new NodeTask(new ExtendedResourceProfile(new NMProfile("low", 2, 4096), 1.0, 2.0), null)
+ fgsNMTask.setHostname("test_fgs_hostname")
+ cgsNMTask.setHostname("test_cgs_hostname")
+ tasks << fgsNMTask << cgsNMTask
+ state.getActiveTasksByType(NodeManagerConfiguration.NM_TASK_PREFIX) >> tasks
+
+ expect:
+ returnValue == SchedulerUtils.isEligibleForFineGrainedScaling(hostName, state)
+
+ where:
+ hostName | returnValue
+ "test_fgs_hostname" | true
+ "test_cgs_hostname" | false
+ "blah" | false
+ "" | false
+ null | false
+ }
+
+ ArrayList<NodeTask> createNodeTaskList(String... hostnames) {
+ def list = []
+ hostnames.each { hostname ->
+ list << createNodeTask(hostname)
+ }
+ return list
+ }
+
+
+ NodeTask createNodeTask(String hostname) {
+ def node = new NodeTask(new ExtendedResourceProfile(new NMProfile("", 1, 1), 1.0, 1.0), null)
+ node.hostname = hostname
+ node.taskPrefix = "nm"
+ node
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TMSTaskFactoryImpl.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TMSTaskFactoryImpl.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TMSTaskFactoryImpl.java
new file mode 100644
index 0000000..89edfe9
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TMSTaskFactoryImpl.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad.scheduler;
+
+import javax.inject.Inject;
+
+import org.apache.mesos.Protos.CommandInfo;
+import org.apache.mesos.Protos.ExecutorInfo;
+import org.apache.mesos.Protos.FrameworkID;
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.TaskID;
+import org.apache.mesos.Protos.TaskInfo;
+
+import org.apache.myriad.configuration.MyriadConfiguration;
+
+/**
+ * Test implementation of TaskFactory
+ */
+public class TMSTaskFactoryImpl implements TaskFactory {
+
+ private MyriadConfiguration cfg;
+ private TaskUtils taskUtils;
+
+ @Inject
+ public TMSTaskFactoryImpl(MyriadConfiguration cfg, TaskUtils taskUtils) {
+ this.setCfg(cfg);
+ this.setTaskUtils(taskUtils);
+ }
+
+ @Override
+ public TaskInfo createTask(Offer offer, FrameworkID frameworkId, TaskID taskId, org.apache.myriad.state.NodeTask nodeTask) {
+ return null;
+ }
+
+ public MyriadConfiguration getCfg() {
+ return cfg;
+ }
+
+ public void setCfg(MyriadConfiguration cfg) {
+ this.cfg = cfg;
+ }
+
+ public TaskUtils getTaskUtils() {
+ return taskUtils;
+ }
+
+ public void setTaskUtils(TaskUtils taskUtils) {
+ this.taskUtils = taskUtils;
+ }
+
+ @Override
+ public ExecutorInfo getExecutorInfoForSlave(FrameworkID frameworkId, Offer offer, CommandInfo commandInfo) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestMyriadScheduler.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestMyriadScheduler.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestMyriadScheduler.java
new file mode 100644
index 0000000..d7abb3d
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestMyriadScheduler.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad.scheduler;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+//import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+//import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+//import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+//import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+//import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.After;
+import org.junit.Before;
+//import org.junit.Test;
+
+//import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests myriad scheduler.
+ */
+public class TestMyriadScheduler {
+ protected Configuration conf;
+ protected FairScheduler scheduler;
+ protected ResourceManager resourceManager;
+
+ @Before
+ public void setUp() throws IOException {
+ scheduler = new org.apache.myriad.scheduler.yarn.MyriadFairScheduler();
+ conf = createConfiguration();
+ resourceManager = new ResourceManager();
+ resourceManager.init(conf);
+
+ // TODO: This test should really be using MockRM. For now starting stuff
+ // that is needed at a bare minimum.
+ ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
+ resourceManager.getRMContext().getStateStore().start();
+
+ // to initialize the master key
+ resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
+
+ scheduler.setRMContext(resourceManager.getRMContext());
+ scheduler.init(conf);
+ scheduler.start();
+ }
+
+ @After
+ public void tearDown() {
+ if (scheduler != null) {
+ scheduler.stop();
+ scheduler = null;
+ }
+
+ if (resourceManager != null) {
+ resourceManager.stop();
+ resourceManager = null;
+ }
+ }
+
+ public Configuration createConfiguration() {
+ Configuration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
+ conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1024);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240);
+ return conf;
+ }
+/*
+ @Test
+ public void testClusterMemory() throws Exception {
+ // Add a node
+ RMNode node1 =
+ MockNodes
+ .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+ assertEquals(1024, scheduler.getClusterResource().getMemory());
+
+ // Add another node
+ RMNode node2 =
+ MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2");
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+ assertEquals(1536, scheduler.getClusterResource().getMemory());
+
+ // Remove the first node
+ NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent3);
+ assertEquals(512, scheduler.getClusterResource().getMemory());
+ }
+ */
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
new file mode 100644
index 0000000..080a76f
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad.scheduler;
+
+import static org.junit.Assert.*;
+
+import org.apache.mesos.Protos.CommandInfo;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.myriad.configuration.MyriadConfiguration;
+import org.apache.myriad.scheduler.TaskFactory.NMTaskFactoryImpl;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ * Class to test CommandLine generation
+ */
+public class TestServiceCommandLine {
+
+ static MyriadConfiguration cfg;
+
+ static String toJHSCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . &&" +
+ " cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\";" +
+ "sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver";
+
+ static String toCompare = "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;";
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class);
+
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ }
+
+ @Test
+ public void testJHSCommandLineGeneration() throws Exception {
+ ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null);
+ String executorCmd = "$YARN_HOME/bin/mapred historyserver";
+ ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0);
+
+ CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd);
+
+ assertTrue(cInfo.getValue().startsWith(toCompare));
+ }
+
+ @Test
+ public void testNMCommandLineGeneration() throws Exception {
+ Long[] ports = new Long[]{1L, 2L, 3L, 4L};
+ NMPorts nmPorts = new NMPorts(ports);
+
+ ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L, 15L), 3.0, 5.0);
+
+ ExecutorCommandLineGenerator clGenerator = new DownloadNMExecutorCLGenImpl(cfg, "hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz");
+ NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator);
+
+ CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts);
+
+ assertTrue(cInfo.getValue().startsWith(toCompare));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
new file mode 100644
index 0000000..f58e1ba
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.myriad.scheduler;
+
+import static org.junit.Assert.*;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.myriad.configuration.MyriadBadConfigurationException;
+import org.apache.myriad.configuration.MyriadConfiguration;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+/**
+ * Tests for TaskUtils
+ */
+public class TestTaskUtils {
+
+ static MyriadConfiguration cfg;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), MyriadConfiguration.class);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ }
+
+ @Test
+ public void testGetResource() {
+ TaskUtils taskUtils = new TaskUtils(cfg);
+
+ NMProfile fooProfile = new NMProfile("abc", 1L, 1000L);
+ try {
+ taskUtils.getAuxTaskCpus(fooProfile, "foo");
+ fail("Should not complete sucessfully for foo");
+ } catch (MyriadBadConfigurationException e) {
+ // success
+ }
+
+ try {
+ double cpu = taskUtils.getAuxTaskCpus(fooProfile, "jobhistory");
+ assertTrue(cpu > 0.0);
+ } catch (MyriadBadConfigurationException e) {
+ fail("cpu should be defined for jobhistory");
+ }
+ }
+
+ @Test
+ public void testServiceResourceProfile() throws Exception {
+ // testing custom deserializer
+
+ Gson gson = new GsonBuilder().registerTypeAdapter(ServiceResourceProfile.class, new ServiceResourceProfile.CustomDeserializer()).create();
+
+
+ ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0);
+
+ String parentStr = gson.toJson(parentProfile);
+ ServiceResourceProfile processedProfile = gson.fromJson(parentStr, ServiceResourceProfile.class);
+
+ assertTrue(processedProfile.getClass().equals(ServiceResourceProfile.class));
+ assertTrue(processedProfile.toString().equalsIgnoreCase(parentStr));
+
+ ServiceResourceProfile childProfile = new ExtendedResourceProfile(new NMProfile("bcd", 5L, 15L), 2.0, 7.0);
+
+ String childStr = gson.toJson(childProfile);
+ ServiceResourceProfile processedChildProfile = gson.fromJson(childStr, ServiceResourceProfile.class);
+
+ assertTrue(processedChildProfile instanceof ExtendedResourceProfile);
+ assertTrue(processedChildProfile.toString().equalsIgnoreCase(childStr));
+ }
+
+ @Test
+
+ public void testStackTrace() {
+
+ new Throwable().printStackTrace();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/constraints/LikeConstraintSpec.groovy
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/constraints/LikeConstraintSpec.groovy b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/constraints/LikeConstraintSpec.groovy
new file mode 100644
index 0000000..b56d23b
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/constraints/LikeConstraintSpec.groovy
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad.scheduler.constraints
+
+import com.google.common.collect.Lists
+import org.apache.mesos.Protos
+import spock.lang.Specification
+
+import static org.apache.mesos.Protos.Value.Text
+import static org.apache.mesos.Protos.Value.Type.TEXT
+
+/**
+ *
+ * Test for LikeConstraint
+ *
+ */
+class LikeConstraintSpec extends Specification {
+
+ def "is matching host name"() {
+ given:
+ def constraint = new LikeConstraint("hostname", "host-[0-9]*.example.com")
+
+ expect:
+ returnValue == constraint.matchesHostName(inputHostName)
+
+ where:
+ inputHostName | returnValue
+ null | false
+ "" | false
+ "blah-blue" | false
+ "host-12.example.com" | true
+ "host-1.example.com" | true
+ "host-2.example.com" | true
+ }
+
+ def "is matching dfs attribute"() {
+ given:
+ def constraint = new LikeConstraint("dfs", "true")
+
+ expect:
+ returnValue == constraint.matchesSlaveAttributes(attributes)
+
+ where:
+ attributes | returnValue
+ null | false
+ Lists.newArrayList() | false
+ Lists.newArrayList(getTextAttribute("dfs", "")) | false
+ Lists.newArrayList(getTextAttribute("dfs", "false")) | false
+ Lists.newArrayList(getTextAttribute("Distributed FS", "true")) | false
+ Lists.newArrayList(getTextAttribute("dfs", "true")) | true
+ Lists.newArrayList(getTextAttribute("dfs", "true"),
+ getTextAttribute("random", "random value")) | true
+ }
+
+ def "equals"() {
+ given:
+ def constraint1 = new LikeConstraint("hostname", "perfnode13[3-4].perf.lab")
+ def constraint2 = new LikeConstraint("hostname", "perfnode13[3-4].perf.lab")
+ def constraint3 = new LikeConstraint("hostname", "perfnode133.perf.lab")
+
+ expect:
+ constraint1.equals(constraint2)
+ !constraint1.equals(constraint3)
+ !constraint2.equals(constraint3)
+ }
+
+ private static Protos.Attribute getTextAttribute(String name, String value) {
+ Protos.Attribute.newBuilder()
+ .setName(name)
+ .setType(TEXT)
+ .setText(Text.newBuilder()
+ .setValue(value))
+ .build()
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
new file mode 100644
index 0000000..afd02be
--- /dev/null
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad.scheduler.fgs
+
+import org.apache.myriad.configuration.MyriadConfiguration
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
+import org.apache.hadoop.yarn.api.records.*
+import org.apache.hadoop.yarn.event.Dispatcher
+import org.apache.hadoop.yarn.event.EventHandler
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode
+import org.apache.hadoop.yarn.util.resource.Resources
+import org.apache.mesos.Protos
+import org.apache.mesos.SchedulerDriver
+import org.apache.myriad.scheduler.MyriadDriver
+import spock.lang.Specification
+
+import java.util.concurrent.ConcurrentHashMap
+
+/**
+ *
+ * Base class for testing Fine Grained Scaling.
+ *
+ */
+class FGSTestBaseSpec extends Specification {
+ def nodeStore = new NodeStore()
+ def mesosDriver = Mock(SchedulerDriver)
+ def myriadDriver = new MyriadDriver(mesosDriver)
+ def offerLifecycleManager = new OfferLifecycleManager(nodeStore, myriadDriver)
+
+ def cfg = new MyriadConfiguration()
+
+ void setup() {
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory())
+ cfg = mapper.readValue(
+ Thread.currentThread().getContextClassLoader().getResource("myriad-config-default.yml"),
+ MyriadConfiguration.class)
+ }
+/******************* Nodes Related ****************/
+
+ def rmNodes = new ConcurrentHashMap<NodeId, RMNode>()
+
+ RMNode getRMNode(int cpu, int mem, String host, Protos.SlaveID slaveId) {
+ RMNode rmNode = MockNodes.newNodeInfo(0, Resources.createResource(mem, cpu), 0, host)
+ if (rmNodes[rmNode.getNodeID()]) {
+ throw new IllegalArgumentException("Node with hostname: " + host + " already exists")
+ }
+ rmNodes.put(rmNode.getNodeID(), rmNode)
+ nodeStore.add(getSchedulerNode(rmNode))
+ def node = nodeStore.getNode(host)
+ node.setSlaveId(slaveId)
+
+ return rmNode
+ }
+
+ SchedulerNode getSchedulerNode(RMNode rmNode) {
+ SchedulerNode schedulerNode = new SchedulerNode(rmNode, false) {
+
+ @Override
+ void reserveResource(SchedulerApplicationAttempt attempt, Priority priority, RMContainer container) {
+ }
+
+ @Override
+ void unreserveResource(SchedulerApplicationAttempt attempt) {
+ }
+ }
+ return schedulerNode
+ }
+
+ /******************* RMContext Related ****************/
+
+ def publisher = Mock(SystemMetricsPublisher) {}
+ def writer = Mock(RMApplicationHistoryWriter) {}
+ def handler = Mock(EventHandler) {}
+
+ def dispatcher = Mock(Dispatcher) {
+ getEventHandler() >> handler
+ }
+
+ def rmContext = Mock(RMContext) {
+ getDispatcher() >> dispatcher
+ getRMApplicationHistoryWriter() >> writer
+ getSystemMetricsPublisher() >> publisher
+ getRMNodes() >> rmNodes
+ }
+
+ /******************* Offers Related ****************/
+
+ Protos.Offer addOfferToFeed(Protos.SlaveID slaveID, String host, int cpu, int mem) {
+ def offer = Protos.Offer.newBuilder()
+ .setId(Protos.OfferID.newBuilder().setValue("test_offer_id"))
+ .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("test_framework_id"))
+ .setSlaveId(slaveID)
+ .setHostname(host)
+ .addResources(Protos.Resource.newBuilder()
+ .setName("cpus")
+ .setScalar(Protos.Value.Scalar.newBuilder().setValue(cpu))
+ .setType(Protos.Value.Type.SCALAR).build())
+ .addResources(Protos.Resource.newBuilder()
+ .setName("mem")
+ .setScalar(Protos.Value.Scalar.newBuilder().setValue(mem))
+ .setType(Protos.Value.Type.SCALAR).build())
+ .build()
+ offerLifecycleManager.addOffers(offer)
+ return offer
+ }
+
+ /******************* Containers Related ****************/
+
+ class FGSContainer {
+ ContainerId containerId
+ Container container
+ RMContainer rmContainer
+ ContainerStatus containerStatus
+ }
+
+ def fgsContainers = new HashMap<>()
+
+ AbstractYarnScheduler yarnScheduler = Mock(AbstractYarnScheduler) {
+ getRMContainer(_ as ContainerId) >> { ContainerId cid -> fgsContainers.get(cid).rmContainer }
+ }
+
+ FGSContainer getFGSContainer(RMNode node, int cid, int cpu, int mem, ContainerState state) {
+ FGSContainer fgsContainer = createFGSContainer(node, cid, cpu, mem, state)
+ if (!fgsContainers[fgsContainer.containerId]) {
+ fgsContainers[fgsContainer.containerId] = fgsContainer
+ }
+ return fgsContainer
+ }
+
+ private FGSContainer createFGSContainer(RMNode node, int cid, int cpu, int mem, ContainerState state) {
+ ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(123456789, 1), 1), cid)
+ FGSContainer fgsContainer = new FGSContainer()
+ fgsContainer.containerId = containerId
+ fgsContainer.container = Container.newInstance(containerId, node.getNodeID(), node.getHttpAddress(),
+ Resources.createResource(mem, cpu), null, null)
+ fgsContainer.rmContainer = new RMContainerImpl(fgsContainer.container, containerId.getApplicationAttemptId(),
+ node.getNodeID(), "user1", rmContext)
+ nodeStore.getNode(node.getNodeID().getHost()).getNode().allocateContainer(fgsContainer.rmContainer)
+ fgsContainer.containerStatus = ContainerStatus.newInstance(containerId, state, "", 0)
+ return fgsContainer
+ }
+
+}