You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2015/05/20 18:05:08 UTC
svn commit: r1680618 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm:
ProcessManagerComponent.java helper/ helper/DuccWorkHelper.java
Author: degenaro
Date: Wed May 20 16:05:08 2015
New Revision: 1680618
URL: http://svn.apache.org/r1680618
Log:
UIMA-4394 DUCC Orchestrator (OR) reduce publication size comprising command line & especially classpath
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/helper/
uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/helper/DuccWorkHelper.java (with props)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java?rev=1680618&r1=1680617&r2=1680618&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java Wed May 20 16:05:08 2015
@@ -30,6 +30,7 @@ import org.apache.uima.ducc.common.compo
import org.apache.uima.ducc.common.main.DuccService;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.pm.helper.DuccWorkHelper;
import org.apache.uima.ducc.transport.cmdline.ICommandLine;
import org.apache.uima.ducc.transport.dispatcher.DuccEventDispatcher;
import org.apache.uima.ducc.transport.event.DuccEvent;
@@ -45,6 +46,8 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
import org.apache.uima.ducc.transport.event.common.IDuccUnits.MemoryUnits;
import org.apache.uima.ducc.transport.event.common.IDuccWork;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkExecutable;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
import org.apache.uima.ducc.transport.event.common.ProcessMemoryAssignment;
/**
@@ -62,6 +65,9 @@ implements ProcessManager {
private static String header;
private static String tbl=String.format("%1$-158s"," ").replace(" ", "-");
public static DuccLogger logger = new DuccLogger(ProcessManagerComponent.class, DuccComponent);
+
+ private static DuccWorkHelper dwHelper = null;
+
// Dispatch component used to send messages to remote Agents
private DuccEventDispatcher eventDispatcher;
private int shareQuantum;
@@ -82,6 +88,7 @@ implements ProcessManager {
String.format(jobHeaderFormat,jobHeaderArray[0],jobHeaderArray[1],jobHeaderArray[2],
jobHeaderArray[3],jobHeaderArray[4],jobHeaderArray[5],jobHeaderArray[6],
jobHeaderArray[7],jobHeaderArray[8]+"\n");
+ dwHelper = new DuccWorkHelper();
}
public void start(DuccService service) throws Exception {
super.start(service, null);
@@ -117,6 +124,20 @@ implements ProcessManager {
return shares;
}
+ private String getCmdLine(ICommandLine iCommandLine) {
+ StringBuffer sb = new StringBuffer();
+ if(iCommandLine != null) {
+ String[] commandLine = iCommandLine.getCommandLine();
+ if(commandLine != null) {
+ for(String item : commandLine) {
+ sb.append(item);
+ sb.append(" ");
+ }
+ }
+ }
+ return sb.toString();
+ }
+
public void dispatchStateUpdateToAgents(ConcurrentHashMap<DuccId, IDuccWork> workMap, long sequence) {
String methodName="dispatchStateUpdateToAgents";
try {
@@ -161,26 +182,50 @@ implements ProcessManager {
logger.debug(methodName,dcj.getDuccId(),"--------------- User Requested Memory For Process:"+dcj.getSchedulingInfo().getShareMemorySize()+dcj.getSchedulingInfo().getShareMemoryUnits()+" PM Calculated Memory Assignment of:"+processAdjustedMemorySize);
ICommandLine driverCmdLine = null;
+ ICommandLine processCmdLine = null;
IDuccProcess driverProcess = null;
+ IDuccWork dw = null;
+
switch(dcj.getDuccType()) {
case Job:
-
- driverCmdLine = dcj.getDriver().getCommandLine();
+ logger.debug(methodName, dcj.getDuccId(), "case: Job");
+ dw = dwHelper.fetch(dcj.getDuccId());
+ IDuccWorkJob job = (IDuccWorkJob) dw;
+ driverCmdLine = job.getDriver().getCommandLine();
+ processCmdLine = job.getCommandLine();
driverProcess = dcj.getDriver().getProcessMap().entrySet().iterator().next().getValue();
-
break;
case Service:
- ///logger.info(methodName,null,"!!!!!!!!!!!!! GOT SERVICE");
- dcj.getCommandLine().addOption("-Dducc.deploy.components=service");
-
+ logger.debug(methodName, dcj.getDuccId(), "case: Service");
+ dw = dwHelper.fetch(dcj.getDuccId());
+ IDuccWorkJob service = (IDuccWorkJob) dw;
+ processCmdLine = service.getCommandLine();
+ processCmdLine.addOption("-Dducc.deploy.components=service");
break;
-
default:
-
+ logger.debug(methodName, dcj.getDuccId(), "case: default");
+ dw = dwHelper.fetch(dcj.getDuccId());
+ if(dw instanceof IDuccWorkExecutable) {
+ IDuccWorkExecutable dwe = (IDuccWorkExecutable) dw;
+ processCmdLine = dwe.getCommandLine();
+ }
+ break;
}
+ String dText = "n/a";
+ if(driverCmdLine != null) {
+ dText = getCmdLine(driverCmdLine);
+ }
+ logger.trace(methodName, dcj.getDuccId(), "driver: "+dText);
+
+ String pText = "n/a";
+ if(processCmdLine != null) {
+ pText = getCmdLine(processCmdLine);
+ }
+ logger.trace(methodName, dcj.getDuccId(), "process: "+pText);
+
jobDeploymentList.add( new DuccJobDeployment(dcj.getDuccId(), driverCmdLine,
- dcj.getCommandLine(),
+ processCmdLine,
dcj.getStandardInfo(),
driverProcess,
pma,
@@ -195,7 +240,7 @@ implements ProcessManager {
logger.debug(methodName,null,"--------------- Added reservation for user:"+userId);
}
}
- }
+ }
logger.info(methodName, null , "---- PM Dispatching DuccJobsStateEvent request to Agent(s) - State Map Size:"+jobDeploymentList.size()+" Reservation List:"+reservationList.size());
DuccJobsStateEvent ev = new DuccJobsStateEvent(DuccEvent.EventType.PM_STATE, jobDeploymentList, reservationList);
ev.setSequence(sequence);
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/helper/DuccWorkHelper.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/helper/DuccWorkHelper.java?rev=1680618&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/helper/DuccWorkHelper.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/helper/DuccWorkHelper.java Wed May 20 16:05:08 2015
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.pm.helper;
+
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.pm.ProcessManager;
+import org.apache.uima.ducc.transport.dispatcher.DuccEventHttpDispatcher;
+import org.apache.uima.ducc.transport.dispatcher.IDuccEventDispatcher;
+import org.apache.uima.ducc.transport.event.DuccWorkReplyEvent;
+import org.apache.uima.ducc.transport.event.DuccWorkRequestEvent;
+import org.apache.uima.ducc.transport.event.common.IDuccWork;
+
+public class DuccWorkHelper {
+
+ public static DuccLogger logger = DuccLogger.getLogger(DuccWorkHelper.class, ProcessManager.DuccComponent);
+
+ private IDuccEventDispatcher dispatcher = null;
+ private DuccId jobid = null;
+ private String orchestrator = "orchestrator";
+
+ public DuccWorkHelper() {
+ init();
+ }
+
+ private void init() {
+ String location = "init";
+ try {
+ String targetUrl = getTargetUrl();
+ dispatcher = new DuccEventHttpDispatcher(targetUrl);
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ }
+
+ private String getServer() {
+ return orchestrator;
+ }
+
+ private String getTargetUrl() {
+ String location = "getTargetUrl";
+ String targetUrl = null;
+ String server = getServer();
+ String host = DuccPropertiesResolver.get("ducc." + server + ".http.node");
+ String port = DuccPropertiesResolver.get("ducc." + server + ".http.port");
+ if ( host == null || port == null ) {
+ String message = "ducc." + server + ".http.node and/or .port not set in ducc.properties";
+ throw new IllegalStateException(message);
+ }
+ targetUrl = "http://" + host + ":" + port + "/" + server.substring(0, 2);
+ logger.info(location, jobid, targetUrl);
+ return targetUrl;
+ }
+
+ public IDuccWork fetch(DuccId duccId) {
+ String location = "fetch";
+ IDuccWork dw = null;
+ if(duccId != null) {
+ DuccWorkRequestEvent dwRequestEvent = new DuccWorkRequestEvent(duccId);
+ DuccWorkReplyEvent dwReplyEvent = null;
+ try {
+ dwReplyEvent = (DuccWorkReplyEvent) dispatcher.dispatchAndWaitForDuccReply(dwRequestEvent);
+ if(dwReplyEvent != null) {
+ dw = dwReplyEvent.getDw();
+ if(dw == null) {
+ logger.debug(location, duccId, "value is null");
+ }
+ else {
+ logger.debug(location, duccId, "state is "+dw.getStateObject());
+ }
+ }
+ else {
+ logger.debug(location, duccId, "reply is null");
+ }
+ }
+ catch (Exception e) {
+ logger.error(location, duccId, e);
+ }
+ }
+ else {
+ logger.debug(location, duccId, "key is null");
+ }
+ return dw;
+ }
+
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/helper/DuccWorkHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/helper/DuccWorkHelper.java
------------------------------------------------------------------------------
svn:mime-type = text/plain