You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2016/06/30 12:13:01 UTC
svn commit: r1750783 - in /uima/uima-ducc/trunk:
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/
uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/
uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/
Author: degenaro
Date: Thu Jun 30 12:13:01 2016
New Revision: 1750783
URL: http://svn.apache.org/viewvc?rev=1750783&view=rev
Log:
UIMA-4616 The uima-ducc-user jar should be added to the head of the user's classpath
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
Modified: uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java Thu Jun 30 12:13:01 2016
@@ -31,6 +31,9 @@ public interface IDuccUser {
//
DUCC_IP("DUCC_IP"),
DUCC_NODENAME("DUCC_NODENAME"),
+ //
+ DUCC_USER_CP_PREPEND("DUCC_USER_CP_PREPEND"),
+ //
USER("USER"),
;
Modified: uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex (original)
+++ uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex Thu Jun 30 12:13:01 2016
@@ -127,6 +127,18 @@
\\Note: When used as a CLI option, the environment string must usually be
quoted to protect it from the shell.
+
+ The following cause special runtime behavior.
+ They are considered experimental and are not guaranteed
+ to be effective from release to release.
+
+ \begin{enumerate}
+ \item[DUCC\_USER\_CP\_PREPEND {[path-to-ducc-jars-and-classes]} ]
+ If specified, this path is used to supply the DUCC classes required for running
+ the Job Driver and Job Process(es), normally set to $DUCC_HOME/lib/uima-ducc/users/*.
+ \end{enumerate}
+
+
\item[$--$help ]
Prints the usage text to the console.
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java Thu Jun 30 12:13:01 2016
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.uima.ducc.common.IDuccEnv;
+import org.apache.uima.ducc.common.IDuccUser;
import org.apache.uima.ducc.common.NodeIdentity;
import org.apache.uima.ducc.common.config.CommonConfiguration;
import org.apache.uima.ducc.common.container.FlagsHelper;
@@ -219,10 +220,21 @@ public class JobFactory implements IJobF
}
}
- private String addUimaDucc(String cp) {
+ private String getPrependUserCP(DuccId jobid, JobRequestProperties jobSpec) {
+ String envKey = IDuccUser.EnvironmentVariable.DUCC_USER_CP_PREPEND.value();
+ String prependDefault = IDuccEnv.DUCC_HOME+File.separator+"lib"+File.separator+"uima-ducc"+File.separator+"user"+File.separator+"*";
+ String retVal = JobFactoryHelper.getEnvVal(jobid, jobSpec, envKey, prependDefault);
+ return retVal;
+ }
+
+ /**
+ * @param prependCP - DUCC classes needed to run JD/JP
+ * @param cp - user classes needed to run JD/JP
+ * @return the entirety of classes needed to run JD/JP: DUCC first, user second
+ */
+ private String addUimaDucc(String prependCP, String cp) {
StringBuffer sb = new StringBuffer();
- String prepend = IDuccEnv.DUCC_HOME+File.separator+"lib"+File.separator+"uima-ducc"+File.separator+"user"+File.separator+"*";
- sb.append(prepend);
+ sb.append(prependCP);
sb.append(File.pathSeparator);
if(cp != null) {
String tcp = cp.trim();
@@ -277,8 +289,9 @@ public class JobFactory implements IJobF
jcl.addOption(opt);
}
// add userCP
+ String prependUserCP = getPrependUserCP(jobid, jobRequestProperties);
String userCP = jobRequestProperties.getProperty(JobSpecificationProperties.key_classpath);
- userCP = addUimaDucc(userCP);
+ userCP = addUimaDucc(prependUserCP, userCP);
opt = FlagsHelper.Name.UserClasspath.dname()+"="+userCP;
jcl.addOption(opt);
// add WorkItemTimeout
@@ -445,6 +458,7 @@ public class JobFactory implements IJobF
private DuccWorkJob create(CommonConfiguration common, JobRequestProperties jobRequestProperties, DuccWorkJob job) {
String methodName = "create";
jobRequestProperties.normalize();
+ DuccId jobid = job.getDuccId();
DuccType duccType = job.getDuccType();
// Service Deployment Type
if(jobRequestProperties.containsKey(ServiceRequestProperties.key_service_type_custom)) {
@@ -580,8 +594,9 @@ public class JobFactory implements IJobF
dump(job, uimaAggregate);
}
// user CP
+ String prependUserCP = getPrependUserCP(jobid, jobRequestProperties);
String userCP = jobRequestProperties.getProperty(JobSpecificationProperties.key_classpath);
- userCP = addUimaDucc(userCP);
+ userCP = addUimaDucc(prependUserCP, userCP);
pipelineCommandLine.setClasspath(userCP);
// jvm args
String process_jvm_args = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_jvm_args);
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java Thu Jun 30 12:13:01 2016
@@ -18,12 +18,18 @@
*/
package org.apache.uima.ducc.orchestrator.factory;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
+import org.apache.uima.ducc.common.utils.QuotedOptions;
import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.cli.JobRequestProperties;
+import org.apache.uima.ducc.transport.event.cli.JobSpecificationProperties;
public class JobFactoryHelper {
@@ -95,4 +101,73 @@ public class JobFactoryHelper {
logger.debug(location, jobid, retVal);
return retVal;
}
+
+ /**
+ * @param jobid the DuccId for the Job
+ * @param jobSpec the Job specification
+ * @return environment variable map derived from user specified environment variable string
+ */
+ public static Map<String, String> getEnvMap(DuccId jobid, JobRequestProperties jobSpec) {
+ String location = "getEnvMap";
+ Map<String, String> retVal = new HashMap<String,String>();
+ if(jobSpec != null) {
+ String environmentVariables = jobSpec.getProperty(JobSpecificationProperties.key_environment);
+ if(environmentVariables != null) {
+ ArrayList<String> envVarList = QuotedOptions.tokenizeList(environmentVariables, true);
+ try {
+ Map<String, String> envMap = QuotedOptions.parseAssignments(envVarList, 0);
+ if(envMap != null) {
+ retVal = envMap;
+ }
+ }
+ catch (IllegalArgumentException e) {
+ logger.warn(location, jobid,"Invalid environment syntax in: " + environmentVariables);
+ }
+ }
+ else {
+ logger.trace(location, jobid, "environmentVariables="+environmentVariables);
+ }
+ }
+ else {
+ logger.debug(location, jobid, "jobSpec="+jobSpec);
+ }
+ return retVal;
+ }
+
+ /**
+ * @param jobid the DuccId for the Job
+ * @param jobSpec the Job specification
+ * @param envKey the environment variable key (name)
+ * @return the environment variable value for the specified key (name)
+ */
+ public static String getEnvVal(DuccId jobid, JobRequestProperties jobSpec, String envKey) {
+ String location = "getEnvVal";
+ String retVal = null;
+ if(envKey != null) {
+ Map<String, String> envMap = getEnvMap(jobid, jobSpec);
+ retVal = envMap.get(envKey);
+ logger.debug(location, jobid, "envKey="+envKey+" "+"envVal="+retVal);
+ }
+ else {
+ logger.debug(location, jobid, "envKey="+envKey);
+ }
+ return retVal;
+ }
+
+ /**
+ * @param jobid the DuccId for the Job
+ * @param jobSpec the Job specification
+ * @param envKey the environment variable key (name)
+ * @param envVal the environment variable default value
+ * @return the environment variable value for the specified key (name)
+ */
+ public static String getEnvVal(DuccId jobid, JobRequestProperties jobSpec, String envKey, String envVal) {
+ String location = "getEnvVal";
+ String retVal = getEnvVal(jobid, jobSpec, envKey);
+ if(retVal == null) {
+ retVal = envVal;
+ logger.debug(location, jobid, "envKey="+envKey+" "+"envVal="+retVal+" "+"(default)");
+ }
+ return retVal;
+ }
}