You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2015/08/28 22:39:50 UTC
svn commit: r1698413 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org: ./
apache/ apache/uima/ apache/uima/ducc/ apache/uima/ducc/orchestrator/
apache/uima/ducc/orchestrator/jd/
apache/uima/ducc/orchestrator/jd/scheduler/ apa...
Author: degenaro
Date: Fri Aug 28 20:39:49 2015
New Revision: 1698413
URL: http://svn.apache.org/r1698413
Log:
UIMA-2869DUCC Orchestrator (OR) must reject [or queue] job submissions when JD allocation is full, so as to not over-commit c-group
- JD Scheduler test
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java (with props)
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java?rev=1698413&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java Fri Aug 28 20:39:49 2015
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.orchestrator.jd.scheduler.test;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.config.CommonConfiguration;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.orchestrator.ReservationFactory;
+import org.apache.uima.ducc.orchestrator.jd.scheduler.JdHostProperties;
+import org.apache.uima.ducc.transport.event.cli.ReservationRequestProperties;
+import org.apache.uima.ducc.transport.event.cli.ReservationSpecificationProperties;
+import org.apache.uima.ducc.transport.event.common.DuccReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
+
+public class TestHelper {
+
+ private static AtomicInteger jdNumber = new AtomicInteger(0);
+
+ private static AtomicInteger hostNumber = new AtomicInteger(0);
+
+ private static String getNodeIp(int i) {
+ return "11.22.33."+i;
+ }
+
+ private static String getNodeName(int i) {
+ return "node"+i;
+ }
+ private static NodeIdentity getNodeIdentity() {
+ NodeIdentity ni = null;
+ int i = hostNumber.getAndIncrement();
+ try {
+ ni = new NodeIdentity(getNodeIp(i), getNodeName(i));
+ }
+ catch(Exception e) {
+ }
+ return ni;
+ }
+
+ public static IDuccWorkReservation getDWR(JdHostProperties jdHostProperties) {
+ String user = jdHostProperties.getHostUser();
+ String pidAtHost = "n/a";
+ String description = jdHostProperties.getHostDescription();
+ String schedulingClass = jdHostProperties.getHostClass();
+ String memorySize = jdHostProperties.getHostMemorySize();
+ return getDWR(user,pidAtHost,description,schedulingClass,memorySize);
+ }
+
+ public static IDuccWorkReservation getDWR(String user, String pidAtHost, String description, String schedulingClass, String memorySize) {
+ IDuccWorkReservation dwr = null;
+ ReservationFactory rf = ReservationFactory.getInstance();
+ CommonConfiguration cc = null;
+ ReservationRequestProperties rrp = new ReservationRequestProperties();
+ String key;
+ String value;
+ //
+ key = ReservationSpecificationProperties.key_user;
+ value = user;
+ rrp.put(key, value);
+ //
+ key = ReservationSpecificationProperties.key_submitter_pid_at_host;
+ value = pidAtHost;
+ rrp.put(key, value);
+ //
+ key = ReservationSpecificationProperties.key_description;
+ value = description;
+ rrp.put(key, value);
+ //
+ key = ReservationSpecificationProperties.key_scheduling_class;
+ value = schedulingClass;
+ rrp.put(key, value);
+ //
+ key = ReservationSpecificationProperties.key_memory_size;
+ value = memorySize;
+ rrp.put(key, value);
+ //
+ dwr = rf.create(cc, rrp);
+ //
+ assign(dwr);
+ //
+ return dwr;
+ }
+
+ public static DuccId getJdId() {
+ DuccId jdId = new DuccId(jdNumber.getAndIncrement());
+ return jdId;
+ }
+
+ public static void assign(IDuccWorkReservation dwr) {
+ IDuccReservationMap rm = dwr.getReservationMap();
+ DuccId duccId = new DuccId(hostNumber.get());
+ NodeIdentity ni = getNodeIdentity();
+ IDuccReservation dr = new DuccReservation(duccId, ni, 1);
+ rm.addReservation(dr);
+ }
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java?rev=1698413&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java Fri Aug 28 20:39:49 2015
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.orchestrator.jd.scheduler.test;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Random;
+
+import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.config.CommonConfiguration;
+import org.apache.uima.ducc.common.main.DuccService;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.orchestrator.OrchestratorCheckpoint;
+import org.apache.uima.ducc.orchestrator.OrchestratorCommonArea;
+import org.apache.uima.ducc.orchestrator.jd.scheduler.JdHostProperties;
+import org.apache.uima.ducc.orchestrator.jd.scheduler.JdScheduler;
+import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
+import org.apache.uima.ducc.transport.event.common.IDuccState.ReservationState;
+import org.apache.uima.ducc.transport.event.common.IDuccWork;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkMap;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestSuite {
+
+ private static DuccLogger logger = null;
+
+ @Before
+ public void before() {
+ ducc_home();
+ ducc_logger();
+ ducc_config();
+ }
+
+ private void ducc_logger() {
+ logger = new DuccLogger(TestSuite.class);
+ DuccService.setDuccLogger(logger);
+ }
+
+ private void ducc_config() {
+ try {
+ DuccService.setDuccLogger(logger);
+ CommonConfiguration commonConfiguration = new CommonConfiguration();
+ OrchestratorCommonArea.initialize(commonConfiguration);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private String getResource(String name) {
+ String retVal = null;
+ URL urlXml = null;
+ File file = null;
+ String path = null;
+ //
+ urlXml = this.getClass().getResource(name);
+ file = new File(urlXml.getFile());
+ path = file.getAbsolutePath();
+ retVal = path;
+ return retVal;
+ }
+
+ private void ducc_home() {
+ String folder = "/ducc_runtime";
+ String file = "/resources/log4j.xml";
+ String path = getResource(folder+file);
+ String value = path.replace(file, "");
+ String key = "DUCC_HOME";
+ System.setProperty(key, value);
+ }
+
+ @After
+ public void after() {
+ }
+
+ private int seed = 0;
+ private Random random = new Random(seed);
+
+ private void assign(IDuccWorkMap dwm) {
+ for(DuccId shareId : dwm.getReservationKeySet()) {
+ IDuccWork dw = dwm.findDuccWork(shareId);
+ IDuccWorkReservation dwr = (IDuccWorkReservation) dw;
+ switch(dwr.getReservationState()) {
+ case WaitingForResources:
+ TestHelper.assign(dwr);
+ // state: WaitingForResources
+ dwr.stateChange(ReservationState.Assigned);
+ OrchestratorCheckpoint.getInstance().saveState();
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ private void publication(JdScheduler jdScheduler, IDuccWorkMap dwm) {
+ assign(dwm);
+ jdScheduler.handle(dwm);
+ }
+
+ private void randomAssign(IDuccWorkMap dwm) {
+ if(random.nextBoolean()) {
+ assign(dwm);
+ }
+ }
+
+ private void randomPublication(JdScheduler jdScheduler, IDuccWorkMap dwm) {
+ if(random.nextInt(10) < 4) {
+ randomAssign(dwm);
+ jdScheduler.handle(dwm);
+ }
+ }
+
+ @Test
+ public void testJdSchedulerStatic() {
+ try {
+ JdHostProperties jdHostProperties = new JdHostProperties();
+ //
+ JdScheduler jdScheduler = new JdScheduler();
+ jdScheduler.resetAutomanage();
+ //
+ jdScheduler.handle(null);
+ IDuccWorkMap dwm = new DuccWorkMap();
+ jdScheduler.handle(dwm);
+ assertTrue(jdScheduler.countSharesTotal() == 0);
+ assertTrue(jdScheduler.countSharesUp() == 0);
+ //
+ IDuccWorkReservation dwr = null;
+ //
+ dwr = TestHelper.getDWR(jdHostProperties);
+ dwm.addDuccWork(dwr);
+ jdScheduler.handle(dwm);
+ assertTrue(jdScheduler.countSharesTotal() == 1);
+ assertTrue(jdScheduler.countSharesUp() == 0);
+ //
+ dwr = TestHelper.getDWR(jdHostProperties);
+ dwr.setReservationState(ReservationState.Assigned);
+ dwm.addDuccWork(dwr);
+ jdScheduler.handle(dwm);
+ assertTrue(jdScheduler.countSharesTotal() == 2);
+ assertTrue(jdScheduler.countSharesUp() == 1);
+ //
+ dwr = TestHelper.getDWR(jdHostProperties);
+ dwr.setReservationState(ReservationState.Assigned);
+ dwm.addDuccWork(dwr);
+ jdScheduler.handle(dwm);
+ assertTrue(jdScheduler.countSharesTotal() == 3);
+ assertTrue(jdScheduler.countSharesUp() == 2);
+ //
+ dwm.removeDuccWork(dwr.getDuccId());
+ jdScheduler.handle(dwm);
+ assertTrue(jdScheduler.countSharesTotal() == 2);
+ assertTrue(jdScheduler.countSharesUp() == 1);
+ //
+ int total = 1152;
+ assertTrue(jdScheduler.countSlicesTotal() == total);
+ assertTrue(jdScheduler.countSlicesInuse() == 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testJdSchedulerDynamic() {
+ try {
+ DuccId jobId = new DuccId(999);
+ //
+ JdScheduler jdScheduler = new JdScheduler();
+ //
+ jdScheduler.handle(null);
+ assertTrue(jdScheduler.countSharesTotal() == 0);
+ assertTrue(jdScheduler.countSharesUp() == 0);
+
+ IDuccWorkMap dwm = new DuccWorkMap();
+ jdScheduler.handle(dwm);
+ //
+ publication(jdScheduler, dwm);
+ //
+ HashMap<DuccId,DuccId> map = new HashMap<DuccId,DuccId>();
+ for(int i=0; i < 100; i++) {
+ DuccId jdId = TestHelper.getJdId();
+ map.put(jdId, jdId);
+ DuccId jdProcessDuccId = (DuccId) jdId;
+ NodeIdentity nodeIdentity = jdScheduler.allocate(jdProcessDuccId, jobId);
+ assertTrue(nodeIdentity != null);
+ randomPublication(jdScheduler, dwm);
+ }
+ int expectedInuse = 100;
+ int expectedTotal = 576;
+ assertTrue(jdScheduler.countSlicesInuse() == expectedInuse);
+ assertTrue(jdScheduler.countSlicesTotal() == expectedTotal);
+ for(Entry<DuccId, DuccId> entry : map.entrySet()) {
+ DuccId jdId = entry.getKey();
+ DuccId jdProcessDuccId = (DuccId) jdId;
+ jdScheduler.deallocate(jdProcessDuccId, jobId);
+ randomPublication(jdScheduler, dwm);
+ }
+ publication(jdScheduler, dwm);
+ int expectedAvailable = 576;
+ assertTrue(jdScheduler.countSlicesInuse() == 0);
+ assertTrue(jdScheduler.countSlicesTotal() == expectedAvailable);
+ assertTrue(jdScheduler.countSlicesAvailable() == expectedAvailable);
+ //
+ for(int i=0; i < 1000; i++) {
+ DuccId jdId = TestHelper.getJdId();
+ DuccId jdProcessDuccId = (DuccId) jdId;
+ map.put(jdId, jdId);
+ NodeIdentity nodeIdentity = jdScheduler.allocate(jdProcessDuccId, jobId);
+ if(nodeIdentity == null) {
+ publication(jdScheduler, dwm);
+ nodeIdentity = jdScheduler.allocate(jdProcessDuccId, jobId);
+ }
+ //assertTrue(nodeIdentity != null);
+ randomPublication(jdScheduler, dwm);
+ }
+ for(Entry<DuccId, DuccId> entry : map.entrySet()) {
+ DuccId jdId = entry.getKey();
+ DuccId jdProcessDuccId = (DuccId) jdId;
+ jdScheduler.deallocate(jdProcessDuccId, jobId);
+ randomPublication(jdScheduler, dwm);
+ }
+ publication(jdScheduler, dwm);
+ assertTrue(jdScheduler.countSlicesInuse() == 0);
+ assertTrue(jdScheduler.countSlicesTotal() == expectedAvailable);
+ assertTrue(jdScheduler.countSlicesAvailable() == expectedAvailable);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
------------------------------------------------------------------------------
svn:mime-type = text/plain