You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2016/06/30 12:13:01 UTC

svn commit: r1750783 - in /uima/uima-ducc/trunk: uima-ducc-common/src/main/java/org/apache/uima/ducc/common/ uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/

Author: degenaro
Date: Thu Jun 30 12:13:01 2016
New Revision: 1750783

URL: http://svn.apache.org/viewvc?rev=1750783&view=rev
Log:
UIMA-4616 The uima-ducc-user jar should be added to the head of the user's classpath

Modified:
    uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java
    uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
    uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
    uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java

Modified: uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java Thu Jun 30 12:13:01 2016
@@ -31,6 +31,9 @@ public interface IDuccUser {
 		//
 		DUCC_IP("DUCC_IP"),
 		DUCC_NODENAME("DUCC_NODENAME"),
+		//
+		DUCC_USER_CP_PREPEND("DUCC_USER_CP_PREPEND"),
+		//
 		USER("USER"),
 		;
 		

Modified: uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex (original)
+++ uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex Thu Jun 30 12:13:01 2016
@@ -127,6 +127,18 @@
              \\Note: When used as a CLI option, the environment string must usually be
              quoted to protect it from the shell.
 
+   
+             The following cause special runtime behavior. 
+             They are considered experimental and are not guaranteed 
+             to be effective from release to release.
+             
+  \begin{enumerate}
+    \item[DUCC\_USER\_CP\_PREPEND {[path-to-ducc-jars-and-classes]}  ]
+    If specified, this path is used to supply the DUCC classes required for running
+    the Job Driver and Job Process(es), normally set to $DUCC_HOME/lib/uima-ducc/users/*.
+  \end{enumerate}
+
+
            \item[$--$help ]
              Prints the usage text to the console. 
 

Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java Thu Jun 30 12:13:01 2016
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.uima.ducc.common.IDuccEnv;
+import org.apache.uima.ducc.common.IDuccUser;
 import org.apache.uima.ducc.common.NodeIdentity;
 import org.apache.uima.ducc.common.config.CommonConfiguration;
 import org.apache.uima.ducc.common.container.FlagsHelper;
@@ -219,10 +220,21 @@ public class JobFactory implements IJobF
 		}
 	}
 	
-	private String addUimaDucc(String cp) {
+	private String getPrependUserCP(DuccId jobid, JobRequestProperties jobSpec) {
+		String envKey = IDuccUser.EnvironmentVariable.DUCC_USER_CP_PREPEND.value();
+		String prependDefault = IDuccEnv.DUCC_HOME+File.separator+"lib"+File.separator+"uima-ducc"+File.separator+"user"+File.separator+"*";
+		String retVal = JobFactoryHelper.getEnvVal(jobid, jobSpec, envKey, prependDefault);
+		return retVal;
+	}
+	
+	/**
+	 * @param prependCP - DUCC classes needed to run JD/JP
+	 * @param cp - user classes needed to run JD/JP
+	 * @return the entirety of classes needed to run JD/JP: DUCC first, user second
+	 */
+	private String addUimaDucc(String prependCP, String cp) {
 		StringBuffer sb = new StringBuffer();
-		String prepend = IDuccEnv.DUCC_HOME+File.separator+"lib"+File.separator+"uima-ducc"+File.separator+"user"+File.separator+"*";
-		sb.append(prepend);
+		sb.append(prependCP);
 		sb.append(File.pathSeparator);
 		if(cp != null) {
 			String tcp = cp.trim();
@@ -277,8 +289,9 @@ public class JobFactory implements IJobF
 			jcl.addOption(opt);
 		}
 		// add userCP
+		String prependUserCP = getPrependUserCP(jobid, jobRequestProperties);
 		String userCP = jobRequestProperties.getProperty(JobSpecificationProperties.key_classpath);
-		userCP = addUimaDucc(userCP);
+		userCP = addUimaDucc(prependUserCP, userCP);
 		opt = FlagsHelper.Name.UserClasspath.dname()+"="+userCP;
 		jcl.addOption(opt);
 		// add WorkItemTimeout	
@@ -445,6 +458,7 @@ public class JobFactory implements IJobF
 	private DuccWorkJob create(CommonConfiguration common, JobRequestProperties jobRequestProperties, DuccWorkJob job) {
 		String methodName = "create";
 		jobRequestProperties.normalize();
+		DuccId jobid = job.getDuccId();
 		DuccType duccType = job.getDuccType();
         // Service Deployment Type
         if(jobRequestProperties.containsKey(ServiceRequestProperties.key_service_type_custom)) {
@@ -580,8 +594,9 @@ public class JobFactory implements IJobF
 				dump(job, uimaAggregate);
 			}
 			// user CP
+			String prependUserCP = getPrependUserCP(jobid, jobRequestProperties);
 			String userCP = jobRequestProperties.getProperty(JobSpecificationProperties.key_classpath);
-			userCP = addUimaDucc(userCP);
+			userCP = addUimaDucc(prependUserCP, userCP);
 			pipelineCommandLine.setClasspath(userCP);
 			// jvm args
 			String process_jvm_args = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_jvm_args);

Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java Thu Jun 30 12:13:01 2016
@@ -18,12 +18,18 @@
 */
 package org.apache.uima.ducc.orchestrator.factory;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.uima.ducc.common.utils.DuccLogger;
 import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
 import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
+import org.apache.uima.ducc.common.utils.QuotedOptions;
 import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.cli.JobRequestProperties;
+import org.apache.uima.ducc.transport.event.cli.JobSpecificationProperties;
 
 public class JobFactoryHelper {
 	
@@ -95,4 +101,73 @@ public class JobFactoryHelper {
 		logger.debug(location, jobid, retVal);
 		return retVal;
 	}
+	
+	/**
+	 * @param jobid the DuccId for the Job
+	 * @param jobSpec the Job specification
+	 * @return environment variable map derived from user specified environment variable string
+	 */
+	public static Map<String, String> getEnvMap(DuccId jobid, JobRequestProperties jobSpec) {
+		String location = "getEnvMap";
+		Map<String, String> retVal = new HashMap<String,String>();
+		if(jobSpec != null) {
+			String environmentVariables = jobSpec.getProperty(JobSpecificationProperties.key_environment);
+			if(environmentVariables != null) {
+				ArrayList<String> envVarList = QuotedOptions.tokenizeList(environmentVariables, true);
+				try {
+					Map<String, String> envMap = QuotedOptions.parseAssignments(envVarList, 0);
+					if(envMap != null) {
+						retVal = envMap;
+					}
+				} 
+				catch (IllegalArgumentException e) {
+	                logger.warn(location, jobid,"Invalid environment syntax in: " + environmentVariables);
+				}
+			}
+			else {
+				logger.trace(location, jobid, "environmentVariables="+environmentVariables);
+			}
+		}
+		else {
+			logger.debug(location, jobid, "jobSpec="+jobSpec);
+		}
+		return retVal;
+	}
+	
+	/**
+	 * @param jobid the DuccId for the Job
+	 * @param jobSpec the Job specification
+	 * @param envKey the environment variable key (name)
+	 * @return the environment variable value for the specified key (name)
+	 */
+	public static String getEnvVal(DuccId jobid, JobRequestProperties jobSpec, String envKey) {
+		String location = "getEnvVal";
+		String retVal = null;
+		if(envKey != null) {
+			Map<String, String> envMap = getEnvMap(jobid, jobSpec);
+			retVal = envMap.get(envKey);
+			logger.debug(location, jobid, "envKey="+envKey+" "+"envVal="+retVal);
+		}
+		else {
+			logger.debug(location, jobid, "envKey="+envKey);
+		}
+		return retVal;
+	}
+	
+	/**
+	 * @param jobid the DuccId for the Job
+	 * @param jobSpec the Job specification
+	 * @param envKey the environment variable key (name)
+	 * @param envVal the environment variable default value
+	 * @return the environment variable value for the specified key (name)
+	 */
+	public static String getEnvVal(DuccId jobid, JobRequestProperties jobSpec, String envKey, String envVal) {
+		String location = "getEnvVal";
+		String retVal = getEnvVal(jobid, jobSpec, envKey);
+		if(retVal == null) {
+			retVal = envVal;
+			logger.debug(location, jobid, "envKey="+envKey+" "+"envVal="+retVal+" "+"(default)");
+		}
+		return retVal;
+	}
 }