You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2017/11/30 13:25:44 UTC
svn commit: r1816711 - in /uima/uima-ducc/trunk: src/main/resources/
uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/
uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/
uima-ducc-pm/src/main/java...
Author: degenaro
Date: Thu Nov 30 13:25:43 2017
New Revision: 1816711
URL: http://svn.apache.org/viewvc?rev=1816711&view=rev
Log:
UIMA-5599 DUCC JD node allocation and watching improvements
In ducc.properties:
# If the value specified is 0 then no JD allocation will take place
# and all submitted jobs will be rejected.
ducc.jd.share.quantum.reserve.count = 3
Modified:
uima/uima-ducc/trunk/src/main/resources/default.ducc.properties
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java
uima/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/event/ProcessManagerEventListener.java
uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWorkMap.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
Modified: uima/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/src/main/resources/default.ducc.properties (original)
+++ uima/uima-ducc/trunk/src/main/resources/default.ducc.properties Thu Nov 30 13:25:43 2017
@@ -413,7 +413,11 @@ ducc.jd.share.quantum = 400
# The Orchestrator makes Reservation requests to RM to get Reservations
# (Job Driver hosts) each of which is then subdivided into "slices", one
# per JD. This number specifies the number of unused "slices" that should
-# be kept on-hand in anticipation of newly submitted jobs (default 2)
+# be kept on-hand in anticipation of newly submitted jobs (default 2).
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# If the value specified is 0 then no JD allocation will take place
+# and all submitted jobs will be rejected.
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
ducc.jd.share.quantum.reserve.count = 3
# The maximum length of a work-item name returned by CAS.getDocumentText().
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java Thu Nov 30 13:25:43 2017
@@ -127,6 +127,7 @@ public class OrchestratorCommonArea {
String ckpt_setting = dpr.getCachedProperty(DuccPropertiesResolver.ducc_orchestrator_checkpoint);
OrchestratorCheckpoint.getInstance().switchOnOff(ckpt_setting);
OrchestratorCheckpoint.getInstance().restoreState();
+ OrchestratorCheckpoint.getInstance().saveState();
jdScheduler = JdScheduler.getInstance();
try {
historyPersistenceManager = HistoryFactory.getInstance(this.getClass().getName());
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java Thu Nov 30 13:25:43 2017
@@ -512,8 +512,13 @@ implements Orchestrator {
+" "+
messages.fetchLabel("active service count")+activeServices
);
- int jobDriverNodeCount = jdScheduler.getReservationCount();
- workMapCopy.setJobDriverNodeCount(jobDriverNodeCount);
+ if(jdScheduler.isMinimalAllocateRequirementMet()) {
+ workMapCopy.setJobDriverMinimalAllocateRequirementMet();
+ }
+ else {
+ workMapCopy.resetJobDriverMinimalAllocateRequirementMet();
+ }
+ logger.debug(methodName, jobid, "isJobDriverMinimalAllocateRequirementMet="+workMapCopy.isJobDriverMinimalAllocateRequirementMet());
orchestratorStateDuccEvent.setWorkMap(workMapCopy);
//stateManager.prune(workMapCopy);
//healthMonitor.cancelNonViableJobs();
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java Thu Nov 30 13:25:43 2017
@@ -48,7 +48,7 @@ public class JdHelper {
}
private static long parseCount(String value) {
- long count = 0;
+ long count = -1;
try {
String tValue = value.trim();
count = new Long(tValue);
@@ -92,7 +92,7 @@ public class JdHelper {
retVal = parseCount(value);
logger.trace(location, jobid, retVal+" "+"specified");
}
- if(retVal < 1) {
+ if(retVal < 0) {
retVal = SlicesReserveDefault;
logger.trace(location, jobid, retVal+" "+"default");
}
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java Thu Nov 30 13:25:43 2017
@@ -152,9 +152,36 @@ public class JdScheduler {
}
}
+ public boolean isMinimalAllocateRequirementMet() {
+ String location = "isMinimalAllocateRequirementMet";
+ boolean retVal = false;
+ StringBuffer sb = new StringBuffer();
+ long minSlices = getReservationSlicesMinimum();
+ long resCount = getReservationCount();
+ sb.append("minSlices="+minSlices);
+ sb.append(" ");
+ sb.append("resCount="+resCount);
+ String text = sb.toString();
+ logger.info(location, jobid, text);
+ if(minSlices == 0) {
+ retVal = true;
+ }
+ else if(resCount > 0) {
+ retVal = true;
+ }
+ return retVal;
+ }
+
+ // Return the number of Reservation Slices minimum needed for JDs.
+
+ private long getReservationSlicesMinimum() {
+ JdHostProperties jdHostProperties = new JdHostProperties();
+ return getSlicesReserveDesired(jdHostProperties);
+ }
+
// Return the number of Reservations allocated for JDs.
- public int getReservationCount() {
+ private int getReservationCount() {
int count = 0;
for(Entry<DuccId, JdReservation> entry : map.entrySet()) {
JdReservation jdReservation = entry.getValue();
Modified: uima/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/event/ProcessManagerEventListener.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/event/ProcessManagerEventListener.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/event/ProcessManagerEventListener.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/event/ProcessManagerEventListener.java Thu Nov 30 13:25:43 2017
@@ -53,7 +53,7 @@ implements DuccEventDelegateListener {
*/
public void onJobManagerStateUpdate(@Body OrchestratorStateDuccEvent duccEvent) {
// process OR state only if the JD has been assigned
- if ( !duccEvent.getWorkMap().isJobDriverNodeAssigned() ) {
+ if ( !duccEvent.getWorkMap().isJobDriverMinimalAllocateRequirementMet() ) {
((ProcessManagerComponent)processManager).getLogger().info("onJobManagerStateUpdate", null, "Orchestrator JD node not assigned. Ignoring Orchestrator state update");
return;
}
Modified: uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java Thu Nov 30 13:25:43 2017
@@ -537,12 +537,6 @@ public class ServiceManagerComponent
logger.info(methodName, null, "===== Orchestrator State Arrives =====");
- if ( workMap.size() == 0 ) {
- logger.debug(methodName, null, "OR state is empty");
- return;
- }
-
-
// try {
// ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream("/home/challngr/for/jerry/working/incomingWorkMap.obj"));
// oos.writeObject(workMap);
@@ -752,11 +746,17 @@ public class ServiceManagerComponent
logger.info(methodName, null, "SM not initialized, ignoring Orchestrator state update.");
return;
}
+ else {
+ logger.debug(methodName, null, "SM initialized.");
+ }
- if ( ! map.isJobDriverNodeAssigned() ) {
+ if ( ! map.isJobDriverMinimalAllocateRequirementMet() ) {
logger.info(methodName, null, "Orchestrator JD node not assigned, ignoring Orchestrator state update.");
return;
}
+ else {
+ logger.debug(methodName, null, "JD requirements met.");
+ }
orchestrator_alive = true;
epochCounter++;
Modified: uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java Thu Nov 30 13:25:43 2017
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang.SerializationUtils;
@@ -43,7 +44,7 @@ public class DuccWorkMap implements IDuc
private Map<DuccId,IDuccWork> concurrentWorkMap = new ConcurrentHashMap<DuccId,IDuccWork>();
- private AtomicInteger atomicJobDriverNodeCount = new AtomicInteger(0);
+ private AtomicBoolean atomicJobDriverMinimalAllocateRequirementMet = new AtomicBoolean(false);
private AtomicInteger atomicJobCount = new AtomicInteger(0);
private AtomicInteger atomicServiceCount = new AtomicInteger(0);
@@ -60,24 +61,24 @@ public class DuccWorkMap implements IDuc
}
private void init() {
- if(atomicJobDriverNodeCount == null) {
- atomicJobDriverNodeCount = new AtomicInteger(0);
+ if(atomicJobDriverMinimalAllocateRequirementMet == null) {
+ atomicJobDriverMinimalAllocateRequirementMet = new AtomicBoolean(false);
}
}
- public boolean isJobDriverNodeAssigned() {
+ public boolean isJobDriverMinimalAllocateRequirementMet() {
init();
- return atomicJobDriverNodeCount.get() > 0;
+ return atomicJobDriverMinimalAllocateRequirementMet.get();
}
- public int getJobDriverNodeCount() {
+ public void setJobDriverMinimalAllocateRequirementMet() {
init();
- return atomicJobDriverNodeCount.get();
+ atomicJobDriverMinimalAllocateRequirementMet.set(true);
}
- public void setJobDriverNodeCount(int count) {
+ public void resetJobDriverMinimalAllocateRequirementMet() {
init();
- atomicJobDriverNodeCount.set(count);
+ atomicJobDriverMinimalAllocateRequirementMet.set(false);
}
public int getJobCount() {
Modified: uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWorkMap.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWorkMap.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWorkMap.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWorkMap.java Thu Nov 30 13:25:43 2017
@@ -47,9 +47,9 @@ public interface IDuccWorkMap extends Se
public List<DuccWorkJob> getServicesList(List<String> implementors);
public Map<Long,DuccWorkJob> getServicesMap(List<String> implementors);
- public boolean isJobDriverNodeAssigned();
+ public boolean isJobDriverMinimalAllocateRequirementMet();
+
public Map<DuccId,IDuccWork> getMap();
- public int getJobDriverNodeCount();
public void addDuccWork(IDuccWork duccWork);
public void removeDuccWork(DuccId duccId);
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java Thu Nov 30 13:25:43 2017
@@ -1524,8 +1524,8 @@ public class DuccHandlerClassic extends
status = DuccHandlerUtils.up();
}
if(daemonName.equals(DaemonName.Orchestrator)) {
- int jdCount = DuccData.getInstance().getLive().getJobDriverNodeCount();
- if(jdCount == 0) {
+ boolean reqMet = DuccData.getInstance().getLive().isJobDriverMinimalAllocateRequirementMet();
+ if(!reqMet) {
status = DuccHandlerUtils.up_provisional(", pending JD allocation");
}
}
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java?rev=1816711&r1=1816710&r2=1816711&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java Thu Nov 30 13:25:43 2017
@@ -2299,8 +2299,8 @@ public class DuccHandlerJsonFormat exten
status = DuccHandlerUtils.up();
}
if(daemonName.equals(DaemonName.Orchestrator)) {
- int jdCount = DuccData.getInstance().getLive().getJobDriverNodeCount();
- if(jdCount == 0) {
+ boolean reqMet = DuccData.getInstance().getLive().isJobDriverMinimalAllocateRequirementMet();
+ if(!reqMet) {
status = DuccHandlerUtils.up_provisional(", pending JD allocation");
}
}