You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2015/08/28 22:39:50 UTC

svn commit: r1698413 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org: ./ apache/ apache/uima/ apache/uima/ducc/ apache/uima/ducc/orchestrator/ apache/uima/ducc/orchestrator/jd/ apache/uima/ducc/orchestrator/jd/scheduler/ apa...

Author: degenaro
Date: Fri Aug 28 20:39:49 2015
New Revision: 1698413

URL: http://svn.apache.org/r1698413
Log:
UIMA-2869DUCC Orchestrator (OR) must reject [or queue] job submissions when JD allocation is full, so as to not over-commit c-group

- JD Scheduler test

Added:
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java   (with props)
    uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java   (with props)

Added: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java?rev=1698413&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java Fri Aug 28 20:39:49 2015
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.orchestrator.jd.scheduler.test;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.config.CommonConfiguration;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.orchestrator.ReservationFactory;
+import org.apache.uima.ducc.orchestrator.jd.scheduler.JdHostProperties;
+import org.apache.uima.ducc.transport.event.cli.ReservationRequestProperties;
+import org.apache.uima.ducc.transport.event.cli.ReservationSpecificationProperties;
+import org.apache.uima.ducc.transport.event.common.DuccReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
+
+public class TestHelper {
+	
+	private static AtomicInteger jdNumber = new AtomicInteger(0);
+	
+	private static AtomicInteger hostNumber = new AtomicInteger(0);
+	
+	private static String getNodeIp(int i) {
+		return "11.22.33."+i;
+	}
+
+	private static String getNodeName(int i) {
+		return "node"+i;
+	}
+	private static NodeIdentity getNodeIdentity() {
+		NodeIdentity ni = null;
+		int i = hostNumber.getAndIncrement();
+		try {
+			ni = new NodeIdentity(getNodeIp(i), getNodeName(i));
+		}
+		catch(Exception e) {
+		}
+		return ni;
+	}
+	
+	public static IDuccWorkReservation getDWR(JdHostProperties jdHostProperties) {
+		String user = jdHostProperties.getHostUser();
+		String pidAtHost = "n/a";
+		String description = jdHostProperties.getHostDescription();
+		String schedulingClass = jdHostProperties.getHostClass();
+		String memorySize = jdHostProperties.getHostMemorySize();
+		return getDWR(user,pidAtHost,description,schedulingClass,memorySize);
+	}
+	
+	public static IDuccWorkReservation getDWR(String user, String pidAtHost, String description, String schedulingClass, String memorySize) {
+		IDuccWorkReservation dwr = null;
+		ReservationFactory rf = ReservationFactory.getInstance();
+		CommonConfiguration cc = null;
+		ReservationRequestProperties rrp = new ReservationRequestProperties();
+		String key;
+		String value;
+		//
+		key = ReservationSpecificationProperties.key_user;
+		value = user;
+		rrp.put(key, value);
+		//
+		key = ReservationSpecificationProperties.key_submitter_pid_at_host;
+		value = pidAtHost;
+		rrp.put(key, value);
+		//
+		key = ReservationSpecificationProperties.key_description;
+		value = description;
+		rrp.put(key, value);
+		//
+		key = ReservationSpecificationProperties.key_scheduling_class;
+		value = schedulingClass;
+		rrp.put(key, value);
+		//
+		key = ReservationSpecificationProperties.key_memory_size;
+		value = memorySize;
+		rrp.put(key, value);
+		//
+		dwr = rf.create(cc, rrp);
+		//
+		assign(dwr);
+		//
+		return dwr;		
+	}
+	
+	public static DuccId getJdId() {
+		DuccId jdId = new DuccId(jdNumber.getAndIncrement());
+		return jdId;
+	}
+	
+	public static void assign(IDuccWorkReservation dwr) {
+		IDuccReservationMap rm = dwr.getReservationMap();
+		DuccId duccId = new DuccId(hostNumber.get());
+		NodeIdentity ni = getNodeIdentity();
+		IDuccReservation dr = new DuccReservation(duccId, ni, 1);
+		rm.addReservation(dr);
+	}
+}

Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestHelper.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java?rev=1698413&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java Fri Aug 28 20:39:49 2015
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.orchestrator.jd.scheduler.test;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Random;
+
+import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.config.CommonConfiguration;
+import org.apache.uima.ducc.common.main.DuccService;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.orchestrator.OrchestratorCheckpoint;
+import org.apache.uima.ducc.orchestrator.OrchestratorCommonArea;
+import org.apache.uima.ducc.orchestrator.jd.scheduler.JdHostProperties;
+import org.apache.uima.ducc.orchestrator.jd.scheduler.JdScheduler;
+import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
+import org.apache.uima.ducc.transport.event.common.IDuccState.ReservationState;
+import org.apache.uima.ducc.transport.event.common.IDuccWork;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkMap;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestSuite {
+	
+	private static DuccLogger logger = null;
+	
+	@Before
+	public void before() {
+		ducc_home();
+		ducc_logger();
+		ducc_config();
+	}
+	
+	private void ducc_logger() {
+		logger = new DuccLogger(TestSuite.class);
+		DuccService.setDuccLogger(logger);
+	}
+	
+	private void ducc_config() {
+		try {
+			DuccService.setDuccLogger(logger);
+			CommonConfiguration commonConfiguration = new CommonConfiguration();
+			OrchestratorCommonArea.initialize(commonConfiguration);
+		}
+		catch(Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+
+	private String getResource(String name) {
+		String retVal = null;
+		URL urlXml = null;
+		File file = null;
+		String path = null;
+		//
+		urlXml = this.getClass().getResource(name);
+		file = new File(urlXml.getFile());
+		path = file.getAbsolutePath();
+		retVal = path;
+		return retVal;
+	}
+	
+	private void ducc_home() {
+		String folder = "/ducc_runtime";
+		String file = "/resources/log4j.xml";
+		String path = getResource(folder+file);
+		String value = path.replace(file, "");
+		String key = "DUCC_HOME";
+		System.setProperty(key, value);
+	}
+	
+	@After
+	public void after() {	
+	}
+
+	private int seed = 0;
+	private Random random = new Random(seed);
+	
+	private void assign(IDuccWorkMap dwm) {
+		for(DuccId shareId : dwm.getReservationKeySet()) {
+			IDuccWork dw = dwm.findDuccWork(shareId);
+			IDuccWorkReservation dwr = (IDuccWorkReservation) dw;
+			switch(dwr.getReservationState()) {
+			case WaitingForResources:
+				TestHelper.assign(dwr);
+				// state: WaitingForResources
+				dwr.stateChange(ReservationState.Assigned);
+				OrchestratorCheckpoint.getInstance().saveState();
+				break;
+			default:
+				break;
+			}
+		}
+	}
+	
+	private void publication(JdScheduler jdScheduler, IDuccWorkMap dwm) {
+		assign(dwm);
+		jdScheduler.handle(dwm);
+	}
+	
+	private void randomAssign(IDuccWorkMap dwm) {
+		if(random.nextBoolean()) {
+			assign(dwm);
+		}
+	}
+	
+	private void randomPublication(JdScheduler jdScheduler, IDuccWorkMap dwm) {
+		if(random.nextInt(10) < 4) {
+			randomAssign(dwm);
+			jdScheduler.handle(dwm);
+		}
+	}
+	
+	@Test
+	public void testJdSchedulerStatic() {
+		try {
+			JdHostProperties jdHostProperties = new JdHostProperties();
+			//
+			JdScheduler jdScheduler = new JdScheduler();
+			jdScheduler.resetAutomanage();
+			//
+			jdScheduler.handle(null);
+			IDuccWorkMap dwm = new DuccWorkMap();
+			jdScheduler.handle(dwm);
+			assertTrue(jdScheduler.countSharesTotal() == 0);
+			assertTrue(jdScheduler.countSharesUp() == 0);
+			//
+			IDuccWorkReservation dwr = null;
+			//
+			dwr = TestHelper.getDWR(jdHostProperties);
+			dwm.addDuccWork(dwr);
+			jdScheduler.handle(dwm);
+			assertTrue(jdScheduler.countSharesTotal() == 1);
+			assertTrue(jdScheduler.countSharesUp() == 0);
+			//
+			dwr = TestHelper.getDWR(jdHostProperties);
+			dwr.setReservationState(ReservationState.Assigned);
+			dwm.addDuccWork(dwr);
+			jdScheduler.handle(dwm);
+			assertTrue(jdScheduler.countSharesTotal() == 2);
+			assertTrue(jdScheduler.countSharesUp() == 1);
+			//
+			dwr = TestHelper.getDWR(jdHostProperties);
+			dwr.setReservationState(ReservationState.Assigned);
+			dwm.addDuccWork(dwr);
+			jdScheduler.handle(dwm);
+			assertTrue(jdScheduler.countSharesTotal() == 3);
+			assertTrue(jdScheduler.countSharesUp() == 2);
+			//
+			dwm.removeDuccWork(dwr.getDuccId());
+			jdScheduler.handle(dwm);
+			assertTrue(jdScheduler.countSharesTotal() == 2);
+			assertTrue(jdScheduler.countSharesUp() == 1);
+			//
+			int total = 1152;
+			assertTrue(jdScheduler.countSlicesTotal() == total);
+			assertTrue(jdScheduler.countSlicesInuse() == 0);
+		}
+		catch(Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+	
+	@Test
+	public void testJdSchedulerDynamic() {
+		try {
+			DuccId jobId = new DuccId(999);
+			//
+			JdScheduler jdScheduler = new JdScheduler();
+			//
+			jdScheduler.handle(null);
+			assertTrue(jdScheduler.countSharesTotal() == 0);
+			assertTrue(jdScheduler.countSharesUp() == 0);
+			
+			IDuccWorkMap dwm = new DuccWorkMap();
+			jdScheduler.handle(dwm);
+			//
+			publication(jdScheduler, dwm);
+			//
+			HashMap<DuccId,DuccId> map = new HashMap<DuccId,DuccId>();
+			for(int i=0; i < 100; i++) {
+				DuccId jdId = TestHelper.getJdId();
+				map.put(jdId, jdId);
+				DuccId jdProcessDuccId = (DuccId) jdId;
+				NodeIdentity nodeIdentity = jdScheduler.allocate(jdProcessDuccId, jobId);
+				assertTrue(nodeIdentity != null);
+				randomPublication(jdScheduler, dwm);
+			}
+			int expectedInuse = 100;
+			int expectedTotal = 576;
+			assertTrue(jdScheduler.countSlicesInuse() == expectedInuse);
+			assertTrue(jdScheduler.countSlicesTotal() == expectedTotal);
+			for(Entry<DuccId, DuccId> entry : map.entrySet()) {
+				DuccId jdId = entry.getKey();
+				DuccId jdProcessDuccId = (DuccId) jdId;
+				jdScheduler.deallocate(jdProcessDuccId, jobId);
+				randomPublication(jdScheduler, dwm);
+			}
+			publication(jdScheduler, dwm);
+			int expectedAvailable = 576;
+			assertTrue(jdScheduler.countSlicesInuse() == 0);
+			assertTrue(jdScheduler.countSlicesTotal() == expectedAvailable);
+			assertTrue(jdScheduler.countSlicesAvailable() == expectedAvailable);
+			//
+			for(int i=0; i < 1000; i++) {
+				DuccId jdId = TestHelper.getJdId();
+				DuccId jdProcessDuccId = (DuccId) jdId;
+				map.put(jdId, jdId);
+				NodeIdentity nodeIdentity = jdScheduler.allocate(jdProcessDuccId, jobId);
+				if(nodeIdentity == null) {
+					publication(jdScheduler, dwm);
+					nodeIdentity = jdScheduler.allocate(jdProcessDuccId, jobId);
+				}
+				//assertTrue(nodeIdentity != null);
+				randomPublication(jdScheduler, dwm);
+			}
+			for(Entry<DuccId, DuccId> entry : map.entrySet()) {
+				DuccId jdId = entry.getKey();
+				DuccId jdProcessDuccId = (DuccId) jdId;
+				jdScheduler.deallocate(jdProcessDuccId, jobId);
+				randomPublication(jdScheduler, dwm);
+			}
+			publication(jdScheduler, dwm);
+			assertTrue(jdScheduler.countSlicesInuse() == 0);
+			assertTrue(jdScheduler.countSlicesTotal() == expectedAvailable);
+			assertTrue(jdScheduler.countSlicesAvailable() == expectedAvailable);
+		}
+		catch(Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+	
+}

Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain