You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by bu...@apache.org on 2017/06/15 20:49:26 UTC
svn commit: r1798878 - in /uima/uima-ducc/trunk:
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/
uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/
uima-ducc-container/src/main/java/org/apache/uima/ducc/contain...
Author: burn
Date: Thu Jun 15 20:49:26 2017
New Revision: 1798878
URL: http://svn.apache.org/viewvc?rev=1798878&view=rev
Log:
UIMA-5428 Create temp descriptors in JPs only if the JD-created one is not accessible. Simplify the AE & DD creation process
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/DgenManager.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/classload/ProxyDeployableGeneration.java
uima/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/classloading/TestClassLoading.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jp/JobProcessComponent.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DeployableGenerator.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaAggregate.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaReferenceByName.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaAggregate.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaReferenceByName.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/DeployableGeneration.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/IDeployableGeneration.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/DuccAbstractProcessContainer.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaASProcessContainer.java
uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaProcessContainer.java
uima/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java
Modified: uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java Thu Jun 15 20:49:26 2017
@@ -47,10 +47,6 @@ public class FlagsHelper {
JpCmDescriptor,
JpCmOverrides,
JpDd,
- JpDdBrokerEndpoint("${ducc.queue.name}"), // NOTE - must match the names in UimaAsProcessContainer
- JpDdBrokerURL("${ducc.broker.name}"),
- JpDdDescription,
- JpDdName,
JpThreadCount,
JpType, // { uima, uima-as }
JpProcessorClass("processor.class"),
@@ -234,42 +230,6 @@ public class FlagsHelper {
return Name.JpDd.arg(value);
}
- public String getJpDdBrokerEndpoint() {
- Properties properties = System.getProperties();
- return properties.getProperty(Name.JpDdBrokerEndpoint.pname());
- }
-
- public String getJpDdBrokerEndpointDashD(String value) {
- return Name.JpDdBrokerEndpoint.arg(value);
- }
-
- public String getJpDdBrokerURL() {
- Properties properties = System.getProperties();
- return properties.getProperty(Name.JpDdBrokerURL.pname());
- }
-
- public String getJpDdBrokerURLDashD(String value) {
- return Name.JpDdBrokerURL.arg(value);
- }
-
- public String getJpDdDescription() {
- Properties properties = System.getProperties();
- return properties.getProperty(Name.JpDdDescription.pname());
- }
-
- public String getJpDdDescriptionDashD(String value) {
- return Name.JpDdDescription.arg(value);
- }
-
- public String getJpDdName() {
- Properties properties = System.getProperties();
- return properties.getProperty(Name.JpDdName.pname());
- }
-
- public String getJpDdNameDashD(String value) {
- return Name.JpDdName.arg(value);
- }
-
public String getJpThreadCount() {
Properties properties = System.getProperties();
return properties.getProperty(Name.JpThreadCount.pname());
Modified: uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/DgenManager.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/DgenManager.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/DgenManager.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/DgenManager.java Thu Jun 15 20:49:26 2017
@@ -60,11 +60,7 @@ public class DgenManager {
initialize(
feh.getJobDirectory(),
feh.getJobId(),
- feh.getJpDdName(),
- feh.getJpDdDescription(),
convert2Integer(feh.getJpThreadCount()),
- feh.getJpDdBrokerURL(),
- feh.getJpDdBrokerEndpoint(),
feh.getJpFlowController(),
feh.getJpAeDescriptor(),
convert2List(feh.getJpAeOverrides()),
@@ -78,11 +74,7 @@ public class DgenManager {
public void initialize(
String jobDirectory,
String jobId,
- String dgenName,
- String dgenDescription,
Integer dgenThreadCount,
- String dgenBrokerURL,
- String dgenBrokerEndpoint,
String flowController,
String aeDescriptor,
List<String> aeOverrides,
@@ -95,15 +87,15 @@ public class DgenManager {
try {
proxy = new ProxyDeployableGeneration();
if(referenceByName == null) {
- String value = proxy.generate(jobDirectory, jobId, dgenName, dgenDescription, dgenThreadCount, dgenBrokerURL, dgenBrokerEndpoint, flowController, cmDescriptor, cmOverrides, aeDescriptor, aeOverrides, ccDescriptor, ccOverrides);
+ String value = proxy.generate(jobDirectory, jobId, dgenThreadCount, flowController, cmDescriptor, cmOverrides, aeDescriptor, aeOverrides, ccDescriptor, ccOverrides);
setDeployable(value);
- logger.info(location, null, "dd from parts: "+value);
+ logger.info(location, null, "ae from parts: "+value);
}
else {
String specification = referenceByName.trim();
- // UIMA-5428 No longer convert the DD in the JD ... it will be done in the JP while extracting the scaleout
- setDeployable(specification);
- logger.info(location, null, "dd specified: "+specification+" (will be modified by the JP)");
+ String value = proxy.generate(jobDirectory, jobId, dgenThreadCount, specification);
+ setDeployable(value);
+ logger.info(location, null, "dd generated: "+value);
}
}
catch(ProxyDeployableGenerationException e) {
Modified: uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/classload/ProxyDeployableGeneration.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/classload/ProxyDeployableGeneration.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/classload/ProxyDeployableGeneration.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/dgen/classload/ProxyDeployableGeneration.java Thu Jun 15 20:49:26 2017
@@ -79,11 +79,7 @@ public class ProxyDeployableGeneration {
public String generate(
String directory,
String id,
- String dgenName,
- String dgenDescription,
Integer dgenThreadCount,
- String dgenBrokerURL,
- String dgenEndpoint,
String dgenFlowController,
String cmDescriptor,
List<String> cmOverrides,
@@ -97,11 +93,7 @@ public class ProxyDeployableGeneration {
try {
show("directory", directory);
show("id", id);
- show("dgenName", dgenName);
- show("dgenDescription", dgenDescription);
show("dgenThreadCount", dgenThreadCount);
- show("dgenBrokerURL", dgenBrokerURL);
- show("dgenEndpoint", dgenEndpoint);
show("degnFlowController", dgenFlowController);
show("cmDescriptor", cmDescriptor);
show("cmOverrides", cmOverrides);
@@ -116,11 +108,7 @@ public class ProxyDeployableGeneration {
Class<?>[] parameterTypes = {
String.class, // directory
String.class, // id
- String.class, // dgenName
- String.class, // dgenDescription
Integer.class, // dgenThreadCount
- String.class, // dgenBrokerURL
- String.class, // dgenEndpoint
String.class, // dgenFlowController
String.class, // cmDescriptor
List.class, // cmOverrides
@@ -128,23 +116,22 @@ public class ProxyDeployableGeneration {
List.class, // aeOverrides
String.class, // ccDescriptor
List.class, // ccOverrides
+ Boolean.class, // createUniqueFilename
};
Method method = clazz.getMethod("generate", parameterTypes);
+ Boolean createUniqueFilename = false; // Use the standard name when created by the JD
Object[] args = {
directory,
id,
- dgenName,
- dgenDescription,
dgenThreadCount,
- dgenBrokerURL,
- dgenEndpoint,
dgenFlowController,
cmDescriptor,
cmOverrides,
aeDescriptor,
aeOverrides,
ccDescriptor,
- ccOverrides
+ ccOverrides,
+ createUniqueFilename
};
String dgen = (String) ContextSwitch.call(urlClassLoader, method, instance, args);
show("generated deployment descriptor", dgen);
@@ -161,52 +148,35 @@ public class ProxyDeployableGeneration {
public String generate(
String directory,
String id,
- String dgenName,
- String dgenDescription,
Integer dgenThreadCount,
- String dgenBrokerURL,
- String dgenEndpoint,
- String dgenFlowController,
- String referenceByName
+ String ddName
) throws ProxyException
{
String retVal = null;
try {
show("directory", directory);
show("id", id);
- show("dgenName", dgenName);
- show("dgenDescription", dgenDescription);
show("dgenThreadCount", dgenThreadCount);
- show("dgenBrokerURL", dgenBrokerURL);
- show("dgenEndpoint", dgenEndpoint);
- show("degnFlowController", dgenFlowController);
- show("referenceByName", referenceByName);
+ show("ddName", ddName);
Class<?> clazz = urlClassLoader.loadClass("org.apache.uima.ducc.user.dgen.iface.DeployableGeneration");
Constructor<?> constructor = clazz.getConstructor();
Object[] c_args = new Object[] { };
Object instance = ContextSwitch.construct(urlClassLoader, constructor, c_args);
Class<?>[] parameterTypes = {
- String.class, // directory
- String.class, // id
- String.class, // dgenName
- String.class, // dgenDescription
- Integer.class, // dgenThreadCount
- String.class, // dgenBrokerURL
- String.class, // dgenEndpoint
- String.class, // dgenFlowController
- String.class, // referenceByName
+ String.class, // directory
+ String.class, // id
+ Integer.class, // dgenThreadCount
+ String.class, // ddName
+ Boolean.class, // createUniqueFilename
};
Method method = clazz.getMethod("generate", parameterTypes);
+ Boolean createUniqueFilename = false; // Use the standard name when created by the JD
Object[] args = {
directory,
id,
- dgenName,
- dgenDescription,
dgenThreadCount,
- dgenBrokerURL,
- dgenEndpoint,
- dgenFlowController,
- referenceByName
+ ddName,
+ createUniqueFilename
};
String dgen = (String) ContextSwitch.call(urlClassLoader, method, instance, args);
show("generated deployment descriptor", dgen);
Modified: uima/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/classloading/TestClassLoading.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/classloading/TestClassLoading.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/classloading/TestClassLoading.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/classloading/TestClassLoading.java Thu Jun 15 20:49:26 2017
@@ -146,11 +146,7 @@ public class TestClassLoading extends Te
//
String directory = working.getAbsolutePath();
String id = "12345";
- String dgenName = "name";
- String dgenDescription = "description";
Integer dgenThreadCount = new Integer(1);
- String dgenBrokerURL = "brokerURL";
- String dgenEndpoint = "endpoint";
String dgenFlowController = "flowController";
String cmDescriptor = null;
List<String> cmOverrides = null;
@@ -162,11 +158,7 @@ public class TestClassLoading extends Te
String dgen = proxy.generate(
directory,
id,
- dgenName,
- dgenDescription,
dgenThreadCount,
- dgenBrokerURL,
- dgenEndpoint,
dgenFlowController,
cmDescriptor,
cmOverrides,
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java Thu Jun 15 20:49:26 2017
@@ -320,16 +320,7 @@ public class JobFactory implements IJobF
addDashD(jcl, FlagsHelper.Name.JpCmOverrides, jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_CM_overrides));
// add Jp DD (pieces-parts should be null)
addDashD(jcl, FlagsHelper.Name.JpDd, jobRequestProperties.getProperty(JobSpecificationProperties.key_process_DD));
- // add Jp DD specs
- String name = "DUCC.Job";
- String description = "DUCC.Generated";
- //TODO
- addDashD(jcl, FlagsHelper.Name.JpDdName, name);
- //TODO
- addDashD(jcl, FlagsHelper.Name.JpDdDescription, description);
addDashD(jcl, FlagsHelper.Name.JpThreadCount, jobRequestProperties.getProperty(JobSpecificationProperties.key_process_pipeline_count));
- addDashD(jcl, FlagsHelper.Name.JpDdBrokerURL, FlagsHelper.Name.JpDdBrokerURL.getDefaultValue());
- addDashD(jcl, FlagsHelper.Name.JpDdBrokerEndpoint, FlagsHelper.Name.JpDdBrokerEndpoint.getDefaultValue());
//
Name flagName;
String flagValue;
@@ -629,18 +620,8 @@ public class JobFactory implements IJobF
addDashD(pipelineCommandLine, FlagsHelper.Name.JpType, "uima-as");
// <flags for JP to build DD>
addDashD(pipelineCommandLine, FlagsHelper.Name.JpDd, process_DD);
- // common
addDashD(pipelineCommandLine, FlagsHelper.Name.JobDirectory, jobRequestProperties.getProperty(JobSpecificationProperties.key_log_directory));
- String keyFCRS = "ducc.flow-controller.specifier";
- String valueFCRS = DuccPropertiesResolver.getInstance().getFileProperty(keyFCRS);
- addDashD(pipelineCommandLine, FlagsHelper.Name.JpFlowController, valueFCRS);
- String name = "DUCC.Job";
- String description = "DUCC.Generated";
- addDashD(pipelineCommandLine, FlagsHelper.Name.JpDdName, name);
- addDashD(pipelineCommandLine, FlagsHelper.Name.JpDdDescription, description);
addDashD(pipelineCommandLine, FlagsHelper.Name.JpThreadCount, jobRequestProperties.getProperty(JobSpecificationProperties.key_process_pipeline_count));
- addDashD(pipelineCommandLine, FlagsHelper.Name.JpDdBrokerURL, FlagsHelper.Name.JpDdBrokerURL.getDefaultValue());
- addDashD(pipelineCommandLine, FlagsHelper.Name.JpDdBrokerEndpoint, FlagsHelper.Name.JpDdBrokerEndpoint.getDefaultValue());
// </flags for JP to build DD>
}
else {
@@ -652,18 +633,11 @@ public class JobFactory implements IJobF
addDashD(pipelineCommandLine, FlagsHelper.Name.JpCcOverrides, jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_CC_overrides));
addDashD(pipelineCommandLine, FlagsHelper.Name.JpCmDescriptor, jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_CM));
addDashD(pipelineCommandLine, FlagsHelper.Name.JpCmOverrides, jobRequestProperties.getProperty(JobSpecificationProperties.key_process_descriptor_CM_overrides));
- // common
addDashD(pipelineCommandLine, FlagsHelper.Name.JobDirectory, jobRequestProperties.getProperty(JobSpecificationProperties.key_log_directory));
String keyFCRS = "ducc.flow-controller.specifier";
String valueFCRS = DuccPropertiesResolver.getInstance().getFileProperty(keyFCRS);
addDashD(pipelineCommandLine, FlagsHelper.Name.JpFlowController, valueFCRS);
- String name = "DUCC.Job";
- String description = "DUCC.Generated";
- addDashD(pipelineCommandLine, FlagsHelper.Name.JpDdName, name);
- addDashD(pipelineCommandLine, FlagsHelper.Name.JpDdDescription, description);
addDashD(pipelineCommandLine, FlagsHelper.Name.JpThreadCount, jobRequestProperties.getProperty(JobSpecificationProperties.key_process_pipeline_count));
- addDashD(pipelineCommandLine, FlagsHelper.Name.JpDdBrokerURL, FlagsHelper.Name.JpDdBrokerURL.getDefaultValue());
- addDashD(pipelineCommandLine, FlagsHelper.Name.JpDdBrokerEndpoint, FlagsHelper.Name.JpDdBrokerEndpoint.getDefaultValue());
// </flags for JP to build Aggregate>
}
Modified: uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jp/JobProcessComponent.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jp/JobProcessComponent.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jp/JobProcessComponent.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jp/JobProcessComponent.java Thu Jun 15 20:49:26 2017
@@ -19,6 +19,7 @@
package org.apache.uima.ducc.transport.configuration.jp;
+import java.io.File;
import java.lang.reflect.Method;
import java.net.SocketTimeoutException;
import java.util.Map;
@@ -245,9 +246,16 @@ implements IJobProcessor{
// Set the initialize args for the appropriate container
// Include the specified pipeline count ... if not defined the container will determine it
+ // UIMA-5428 If the DD generated by the JD is not accessible revert to the one specified
+ // by the user ... it will be converted to a temporary file by the JP
String[] jpArgs;
if ("uima-as".equals(jobType)) {
uimaASJob = true; // dd - deployment descriptor. Will use UIMA-AS
+ if (!new File(args[0]).canRead()) {
+ String userdd = FlagsHelper.getInstance().getJpDd();
+ getLogger().info("start", null, "Replacing inaccessible DD "+args[0]+" by the user specified "+userdd);
+ args[0] = userdd;
+ }
jpArgs = new String[] { "-dd", args[0], "-saxonURL", saxonJarPath, "-xslt", dd2SpringXslPath, "-t", jpThreadCount };
} else if ("uima".equals(jobType)) {
// aed - analysis engine descriptor. Will use UIMA core only
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DeployableGenerator.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DeployableGenerator.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DeployableGenerator.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DeployableGenerator.java Thu Jun 15 20:49:26 2017
@@ -23,9 +23,11 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
-import java.util.UUID;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -36,6 +38,7 @@ import javax.xml.transform.stream.Stream
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.ducc.user.common.UimaUtils;
+import org.apache.uima.ducc.user.jp.UimaASProcessContainer;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@@ -45,7 +48,7 @@ public class DeployableGenerator {
private String userLogDir = null;
private Document doc;
private String registryURL;
-
+
public DeployableGenerator(String userLogDir) {
setUserLogDir(userLogDir);
}
@@ -54,22 +57,7 @@ public class DeployableGenerator {
userLogDir = value;
}
- public String generate(IDuccGeneratorUimaDeployableConfiguration configuration, String jobId) throws Exception {
- String retVal = null;
- if(configuration != null) {
- if(configuration instanceof IDuccGeneratorUimaReferenceByName) {
- IDuccGeneratorUimaReferenceByName referrenceByNameConfiguration = (IDuccGeneratorUimaReferenceByName) configuration;
- retVal = generateDd(referrenceByNameConfiguration, jobId);
- }
- else if(configuration instanceof IDuccGeneratorUimaAggregate) {
- IDuccGeneratorUimaAggregate aggregateConfiguration = (IDuccGeneratorUimaAggregate) configuration;
- retVal = generateAe(aggregateConfiguration, jobId);
- }
- }
- return retVal;
- }
-
- private String generateAe(IDuccGeneratorUimaAggregate aggregateConfiguration, String jobId) throws Exception {
+ public String generateAe(IDuccGeneratorUimaAggregate aggregateConfiguration, String jobId, boolean createUniqueFilename) throws Exception {
List<String> descriptorPaths = new ArrayList<String>();
List<List<String>> overrides = new ArrayList<List<String>>();
for( IDuccGeneratorUimaAggregateComponent component: aggregateConfiguration.getComponents()) {
@@ -77,25 +65,17 @@ public class DeployableGenerator {
overrides.add(component.getOverrides());
}
String aed = createAED(
- aggregateConfiguration.getName(),
- aggregateConfiguration.getDescription(),
- aggregateConfiguration.getBrokerURL(),
- aggregateConfiguration.getEndpoint(),
aggregateConfiguration.getFlowController(),
aggregateConfiguration.getThreadCount(),
userLogDir,
- jobId+"-"+"uima-ae-descriptor"+".xml",
- overrides,
- descriptorPaths.toArray(new String[descriptorPaths.size()])
+ createUniqueFilename ? null : jobId+"-"+"uima-ae-descriptor"+".xml",
+ overrides,
+ descriptorPaths.toArray(new String[descriptorPaths.size()])
);
return aed;
}
private static String createAED (
- String name,
- String description,
- String brokerURL,
- String endpoint,
String flowController,
int scaleup,
String directory,
@@ -104,18 +84,25 @@ public class DeployableGenerator {
String... aeDescriptors) throws Exception {
AnalysisEngineDescription aed = UimaUtils.createAggregateDescription(flowController, (scaleup > 1), overrides, aeDescriptors);
- aed.getMetaData().setName(name);
- File file = null;
+ aed.getMetaData().setName("DUCC.job");
File dir = new File(directory);
if (!dir.exists()) {
dir.mkdir();
}
FileOutputStream fos = null;
try {
- file = new File(dir, fname);//+"-uima-ae-descriptor-"+Utils.getPID()+".xml");
+ File file = File.createTempFile("uima-ae-", ".xml", dir);
fos = new FileOutputStream(file);
aed.toXML(fos);
-
+ if (fname == null) { // Use the unique name
+ deleteOnExitCheck(file);
+ return file.getAbsolutePath();
+ }
+ // Use the atomic Files.move method (reportedly better than File:renameTo)
+ Path source = file.toPath();
+ Path target = source.resolveSibling(fname);
+ Files.move(source, target, StandardCopyOption.ATOMIC_MOVE);
+ return target.toString();
}
catch(Exception e) {
throw e;
@@ -125,7 +112,6 @@ public class DeployableGenerator {
fos.close();
}
}
- return file.getAbsolutePath();
}
/*
@@ -133,7 +119,7 @@ public class DeployableGenerator {
* to make it suitable for the JP's internal broker.
* It is also used by the JP code since when running as a "pull" service it will be given an unconverted DD
*/
- private String generateDd(IDuccGeneratorUimaReferenceByName configuration, String jobId) throws Exception {
+ public String generateDd(IDuccGeneratorUimaReferenceByName configuration, String jobId, Boolean createUniqueFilename) throws Exception {
// Create DOM from the DD ... file or class-like name
String location = configuration.getReferenceByName();
org.apache.uima.util.XMLInputSource xmlin = UimaUtils.getXMLInputSource(location); // Reads from FS or classpath
@@ -149,12 +135,12 @@ public class DeployableGenerator {
if (nodes.getLength() > 0) {
element = (Element) nodes.item(0);
// Check if the attributes are correct
- String expected = configuration.getEndpoint();
+ String expected = "${" + UimaASProcessContainer.queuePropertyName + "}";
if ( ! element.getAttribute("endpoint").equals(expected)) {
element.setAttribute("endpoint", expected);
createDescriptor = true;
}
- expected = configuration.getBrokerURL();
+ expected = "${" + UimaASProcessContainer.brokerPropertyName + "}";
if ( ! element.getAttribute("brokerURL").equals(expected)) {
element.setAttribute("brokerURL", expected);
createDescriptor = true;
@@ -168,7 +154,7 @@ public class DeployableGenerator {
}
// Return the original descriptor or the converted one if necessary
- return createDescriptor ? writeDDFile(xml2String(doc), jobId) : location;
+ return createDescriptor ? writeDDFile(xml2String(doc), jobId, createUniqueFilename) : location;
}
/*
@@ -232,24 +218,26 @@ public class DeployableGenerator {
return serializedDD.toString();
}
- private String writeDDFile(String content, String jobId) throws Exception {
+ private String writeDDFile(String content, String jobId, boolean createUniqueFilename) throws Exception {
File dir = new File(userLogDir);
if ( !dir.exists()) {
dir.mkdir();
}
// compose the file name from a basename (from ducc.properties), constant (-uima-as.dd-) and PID
+ // Create as a temp file then rename atomically (unless the JP wants a unique temporary file)
BufferedWriter out = null;
try {
- // using PID of the ducc component process in the DD file name
- UUID uuid = UUID.randomUUID();
- File file1 = new File(dir, uuid.toString());
- File file2 = new File(dir, jobId+"-uima-as-dd"+".xml");
- out = new BufferedWriter(new FileWriter(file1));
+ File file = File.createTempFile("uima-as-dd-", ".xml", dir);
+ out = new BufferedWriter(new FileWriter(file));
out.write(content);
- out.flush();
- file1.renameTo(file2);
- //file1.delete();
- return file2.getAbsolutePath();
+ if (createUniqueFilename) {
+ deleteOnExitCheck(file);
+ return file.getAbsolutePath();
+ }
+ Path source = file.toPath();
+ Path target = source.resolveSibling(jobId+"-uima-as-dd.xml");
+ Files.move(source, target, StandardCopyOption.ATOMIC_MOVE);
+ return target.toString();
} catch( Exception e) {
throw e;
} finally {
@@ -258,4 +246,12 @@ public class DeployableGenerator {
}
}
}
+
+ // Don't delete descriptors if this environment variable is set
+ // (Can't put the key in IDuccUser as that is in the common project)
+ private static void deleteOnExitCheck(File f) {
+ if (System.getenv("DUCC_KEEP_TEMPORARY_DESCRIPTORS") == null) {
+ f.deleteOnExit();
+ }
+ }
}
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaAggregate.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaAggregate.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaAggregate.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaAggregate.java Thu Jun 15 20:49:26 2017
@@ -30,28 +30,11 @@ public class DuccUimaAggregate implement
private List<IDuccGeneratorUimaAggregateComponent> components = new ArrayList<IDuccGeneratorUimaAggregateComponent>();
- private String name;
- private String description;
private int threadCount = 1;
- private String brokerURL;
- private String endpoint;
private String flowController;
- public DuccUimaAggregate(String name, String description, int threadCount, String brokerURL, String endpoint, String flowController) {
- setName(name);
- setDescription(description);
+ public DuccUimaAggregate(int threadCount, String flowController, List<IDuccGeneratorUimaAggregateComponent> components) {
setThreadCount(threadCount);
- setBrokerURL(brokerURL);
- setEndpoint(endpoint);
- setFlowController(flowController);
- }
-
- public DuccUimaAggregate(String name, String description, int threadCount, String brokerURL, String endpoint, String flowController, List<IDuccGeneratorUimaAggregateComponent> components) {
- setName(name);
- setDescription(description);
- setThreadCount(threadCount);
- setBrokerURL(brokerURL);
- setEndpoint(endpoint);
setFlowController(flowController);
setComponents(components);
}
@@ -66,27 +49,6 @@ public class DuccUimaAggregate implement
this.components = components;
}
-
- public String getName() {
- return name;
- }
-
-
- public void setName(String name) {
- this.name = name;
- }
-
-
- public String getDescription() {
- return description;
- }
-
-
- public void setDescription(String description) {
- this.description = description;
- }
-
-
public int getThreadCount() {
return threadCount;
}
@@ -96,27 +58,6 @@ public class DuccUimaAggregate implement
this.threadCount = threadCount;
}
-
- public String getBrokerURL() {
- return brokerURL;
- }
-
-
- public void setBrokerURL(String brokerURL) {
- this.brokerURL = brokerURL;
- }
-
-
- public String getEndpoint() {
- return endpoint;
- }
-
-
- public void setEndpoint(String endpoint) {
- this.endpoint = endpoint;
- }
-
-
public String getFlowController() {
return flowController;
}
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaReferenceByName.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaReferenceByName.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaReferenceByName.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/DuccUimaReferenceByName.java Thu Jun 15 20:49:26 2017
@@ -27,63 +27,21 @@ public class DuccUimaReferenceByName imp
private String referenceByName;
- private String name;
- private String description;
private int threadCount = 1;
- private String brokerURL;
- private String endpoint;
- private String flowController;
-
- public DuccUimaReferenceByName(String name, String description, int threadCount, String brokerURL, String endpoint, String flowController) {
- setName(name);
- setDescription(description);
- setThreadCount(threadCount);
- setBrokerURL(brokerURL);
- setEndpoint(endpoint);
- setFlowController(flowController);
- }
- public DuccUimaReferenceByName(String name, String description, int threadCount, String brokerURL, String endpoint, String flowController, String referenceByName) {
- setName(name);
- setDescription(description);
+ public DuccUimaReferenceByName(int threadCount,String ddName) {
setThreadCount(threadCount);
- setBrokerURL(brokerURL);
- setEndpoint(endpoint);
- setFlowController(flowController);
- setReferenceByName(referenceByName);
+ setReferenceByName(ddName);
}
-
public String getReferenceByName() {
return referenceByName;
}
-
public void setReferenceByName(String referenceByName) {
this.referenceByName = referenceByName;
}
-
- public String getName() {
- return name;
- }
-
-
- public void setName(String name) {
- this.name = name;
- }
-
-
- public String getDescription() {
- return description;
- }
-
-
- public void setDescription(String description) {
- this.description = description;
- }
-
-
public int getThreadCount() {
return threadCount;
}
@@ -93,33 +51,4 @@ public class DuccUimaReferenceByName imp
this.threadCount = threadCount;
}
-
- public String getBrokerURL() {
- return brokerURL;
- }
-
-
- public void setBrokerURL(String brokerURL) {
- this.brokerURL = brokerURL;
- }
-
-
- public String getEndpoint() {
- return endpoint;
- }
-
-
- public void setEndpoint(String endpoint) {
- this.endpoint = endpoint;
- }
-
-
- public String getFlowController() {
- return flowController;
- }
-
-
- public void setFlowController(String flowController) {
- this.flowController = flowController;
- }
}
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaAggregate.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaAggregate.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaAggregate.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaAggregate.java Thu Jun 15 20:49:26 2017
@@ -25,21 +25,9 @@ public interface IDuccGeneratorUimaAggre
public List<IDuccGeneratorUimaAggregateComponent> getComponents();
public void setComponents(List<IDuccGeneratorUimaAggregateComponent> components);
- public String getName();
- public void setName(String name);
-
- public String getDescription();
- public void setDescription(String description);
-
public int getThreadCount();
public void setThreadCount(int threadCount);
- public String getBrokerURL();
- public void setBrokerURL(String brokerURL);
-
- public String getEndpoint();
- public void setEndpoint(String endpoint);
-
public String getFlowController();
public void setFlowController(String flowController);
}
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaReferenceByName.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaReferenceByName.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaReferenceByName.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/IDuccGeneratorUimaReferenceByName.java Thu Jun 15 20:49:26 2017
@@ -23,21 +23,7 @@ public interface IDuccGeneratorUimaRefer
public String getReferenceByName();
public void setReferenceByName(String referenceByName);
- public String getName();
- public void setName(String name);
-
- public String getDescription();
- public void setDescription(String description);
-
public int getThreadCount();
public void setThreadCount(int threadCount);
-
- public String getBrokerURL();
- public void setBrokerURL(String brokerURL);
-
- public String getEndpoint();
- public void setEndpoint(String endpoint);
-
- public String getFlowController();
- public void setFlowController(String flowController);
+
}
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/DeployableGeneration.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/DeployableGeneration.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/DeployableGeneration.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/DeployableGeneration.java Thu Jun 15 20:49:26 2017
@@ -27,7 +27,6 @@ import org.apache.uima.ducc.user.dgen.Du
import org.apache.uima.ducc.user.dgen.DuccUimaAggregateComponent;
import org.apache.uima.ducc.user.dgen.DuccUimaReferenceByName;
import org.apache.uima.ducc.user.dgen.IDuccGeneratorUimaAggregateComponent;
-import org.apache.uima.ducc.user.dgen.IDuccGeneratorUimaDeployableConfiguration;
public class DeployableGeneration implements IDeployableGeneration {
@@ -76,29 +75,22 @@ public class DeployableGeneration implem
public String generate(
String directory,
String id,
- String dgenName,
- String dgenDescription,
Integer dgenThreadCount,
- String dgenBrokerURL,
- String dgenEndpoint,
String dgenFlowController,
String cmDescriptor,
List<String> cmOverrides,
String aeDescriptor,
List<String> aeOverrides,
String ccDescriptor,
- List<String> ccOverrides
+ List<String> ccOverrides,
+ Boolean createUniqueFilename
) throws Exception
{
String retVal = null;
try {
show("directory", directory);
show("id", id);
- show("dgenName", dgenName);
- show("dgenDescription", dgenDescription);
show("dgenThreadCount", dgenThreadCount.toString());
- show("dgenBrokerURL", dgenBrokerURL);
- show("dgenEndpoint", dgenEndpoint);
show("dgenFlowController", dgenFlowController);
show("cmDescriptor", cmDescriptor);
show("cmOverrides", cmOverrides);
@@ -106,14 +98,16 @@ public class DeployableGeneration implem
show("aeOverrides", aeOverrides);
show("ccDescriptor", ccDescriptor);
show("ccOverrides", ccOverrides);
+ show("createUniqueFilename", createUniqueFilename?"true":"false");
String targetDirectory = fabricateTargetDirectoryName(directory, id);
DeployableGenerator deployableGenerator = new DeployableGenerator(targetDirectory);
ArrayList<IDuccGeneratorUimaAggregateComponent> dgenComponents = new ArrayList<IDuccGeneratorUimaAggregateComponent>();
conditionalAddComponent(dgenComponents, cmDescriptor, cmOverrides);
conditionalAddComponent(dgenComponents, aeDescriptor, aeOverrides);
conditionalAddComponent(dgenComponents, ccDescriptor, ccOverrides);
- IDuccGeneratorUimaDeployableConfiguration configuration = new DuccUimaAggregate(dgenName, dgenDescription, dgenThreadCount, dgenBrokerURL, dgenEndpoint, dgenFlowController, dgenComponents);
- retVal = deployableGenerator.generate(configuration, id);
+ DuccUimaAggregate configuration = new DuccUimaAggregate(dgenThreadCount, dgenFlowController, dgenComponents);
+ retVal = deployableGenerator.generateAe(configuration, id, createUniqueFilename);
+
}
catch(Exception e) {
e.printStackTrace();
@@ -126,30 +120,22 @@ public class DeployableGeneration implem
public String generate(
String directory,
String id,
- String dgenName,
- String dgenDescription,
Integer dgenThreadCount,
- String dgenBrokerURL,
- String dgenEndpoint,
- String dgenFlowController,
- String dgenReferenceByName
+ String ddName,
+ Boolean createUniqueFilename
) throws Exception
{
String retVal = null;
try {
show("directory", directory);
show("id", id);
- show("dgenName", dgenName);
- show("dgenDescription", dgenDescription);
show("dgenThreadCount", dgenThreadCount.toString());
- show("dgenBrokerURL", dgenBrokerURL);
- show("dgenEndpoint", dgenEndpoint);
- show("dgenFlowController", dgenFlowController);
- show("dgenReferenceByName", dgenReferenceByName);
+ show("ddName", ddName);
+ show("createUniqueFilename", createUniqueFilename?"true":"false");
String targetDirectory = fabricateTargetDirectoryName(directory, id);
DeployableGenerator deployableGenerator = new DeployableGenerator(targetDirectory);
- IDuccGeneratorUimaDeployableConfiguration configuration = new DuccUimaReferenceByName(dgenName, dgenDescription, dgenThreadCount, dgenBrokerURL, dgenEndpoint, dgenFlowController, dgenReferenceByName);
- retVal = deployableGenerator.generate(configuration, id);
+ DuccUimaReferenceByName configuration = new DuccUimaReferenceByName(dgenThreadCount, ddName);
+ retVal = deployableGenerator.generateDd(configuration, id, createUniqueFilename);
}
catch(Exception e) {
e.printStackTrace();
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/IDeployableGeneration.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/IDeployableGeneration.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/IDeployableGeneration.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/dgen/iface/IDeployableGeneration.java Thu Jun 15 20:49:26 2017
@@ -25,29 +25,22 @@ public interface IDeployableGeneration {
public String generate(
String directory,
String id,
- String dgenName,
- String dgenDescription,
Integer dgenThreadCount,
- String dgenBrokerURL,
- String dgenEndpoint,
String dgenFlowController,
String cm,
List<String> cmOverrides,
String ae,
List<String> aeOverrides,
String cc,
- List<String> ccOverrides
+ List<String> ccOverrides,
+ Boolean createUniqueFilename
) throws Exception;
public String generate(
String directory,
String id,
- String dgenName,
- String dgenDescription,
Integer dgenThreadCount,
- String dgenBrokerURL,
- String dgenEndpoint,
- String dgenFlowController,
- String dgenReferenceByName
+ String ddName,
+ Boolean createUniqueFilename
) throws Exception;
}
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/DuccAbstractProcessContainer.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/DuccAbstractProcessContainer.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/DuccAbstractProcessContainer.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/DuccAbstractProcessContainer.java Thu Jun 15 20:49:26 2017
@@ -251,37 +251,21 @@ public abstract class DuccAbstractProces
return value;
}
- protected int buildDeployable() {
- int retVal = 0;
+ // Build just an AE from parts and return the filename
+ // (DD's are converted in UimaAsProcessContainer.parseDD)
+ protected String buildDeployable() {
try {
dumpSystemProperties();
String directory = getPropertyString("ducc.deploy.JobDirectory");
String id = getPropertyString("ducc.job.id");
- String dgenName = getPropertyString("ducc.deploy.JpDdName");
- String dgenDescription = getPropertyString("ducc.deploy.JpDdDescription");
Integer dgenThreadCount = getPropertyInteger("ducc.deploy.JpThreadCount");
- String dgenBrokerURL = getPropertyString("ducc.deploy.JpDdBrokerURL");
- String dgenEndpoint = getPropertyString("ducc.deploy.JpDdBrokerEndpoint");
String dgenFlowController = getPropertyString("ducc.deploy.JpFlowController");
String jpType = getPropertyString("ducc.deploy.JpType");
if(jpType == null) {
jpType = "uima";
}
if(jpType.equalsIgnoreCase("uima-as")) {
- String dgenReferenceByName = getPropertyString("ducc.deploy.JpDd");
- DeployableGeneration dg = new DeployableGeneration();
- String name = dg.generate(
- directory,
- id,
- dgenName,
- dgenDescription,
- dgenThreadCount,
- dgenBrokerURL,
- dgenEndpoint,
- dgenFlowController,
- dgenReferenceByName
- );
- System.out.println("type="+jpType+" name="+name);
+ System.out.println("ERROR - should not be called for type="+jpType);
}
else {
String cmDescriptor = getPropertyString("ducc.deploy.JpCmDescriptor");
@@ -294,27 +278,23 @@ public abstract class DuccAbstractProces
String name = dg.generate(
directory,
id,
- dgenName,
- dgenDescription,
dgenThreadCount,
- dgenBrokerURL,
- dgenEndpoint,
dgenFlowController,
cmDescriptor,
cmOverrides,
aeDescriptor,
aeOverrides,
ccDescriptor,
- ccOverrides
+ ccOverrides,
+ true // create unique temporary file
);
- System.out.println("type="+jpType+" name="+name);
+ return name;
}
}
catch(Exception e) {
e.printStackTrace();
- retVal = -1;
}
- return retVal;
+ return null;
}
}
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaASProcessContainer.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaASProcessContainer.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaASProcessContainer.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaASProcessContainer.java Thu Jun 15 20:49:26 2017
@@ -48,7 +48,6 @@ import org.apache.uima.ducc.IUser;
import org.apache.uima.ducc.user.common.main.DuccJobService;
import org.apache.uima.ducc.user.dgen.DeployableGenerator;
import org.apache.uima.ducc.user.dgen.DuccUimaReferenceByName;
-import org.apache.uima.ducc.user.dgen.IDuccGeneratorUimaDeployableConfiguration;
import org.apache.uima.util.Level;
import org.apache.uima.util.Logger;
@@ -71,12 +70,11 @@ public class UimaASProcessContainer ext
private volatile boolean threadAffinity=false;
boolean enablePerformanceBreakdownReporting = false;
- // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
- // WARNING - these names are also in FlagsHelper but that is in common so can't be used here
- // Since they are put in the system properties their names should be "reserved"
- private static final String brokerPropertyName = "ducc.broker.name";
- private static final String queuePropertyName = "ducc.queue.name";
-
+ // NOTE - these names are also used in DeployableGenerator when
+ // creating a DD with the appropriate placeholders
+ public static final String brokerPropertyName = "ducc.broker.name";
+ public static final String queuePropertyName = "ducc.queue.name";
+
public boolean useThreadAffinity() {
return threadAffinity;
}
@@ -487,17 +485,12 @@ public class UimaASProcessContainer ext
* @throws Exception
*/
public String parseDD(String ddPath, String threadCount) throws Exception {
- // The queue & broker attributes must reference these system properties
- final String reqdBrokerName = "${" + brokerPropertyName + "}";
- final String reqdQueueName = "${" + queuePropertyName + "}";
-
// For "custom" pull-services must convert the DD so it can be deployed locally
// For JPs this should return the same file, already converted by the JD
String logDir = new File(System.getenv("DUCC_PROCESS_LOG_PREFIX")).getParent();
DeployableGenerator deployableGenerator = new DeployableGenerator(logDir);
- IDuccGeneratorUimaDeployableConfiguration configuration =
- new DuccUimaReferenceByName(null, null, 0, reqdBrokerName, reqdQueueName, null, ddPath);
- String ddNew = deployableGenerator.generate(configuration, System.getenv("DUCC_JOBID"));
+ DuccUimaReferenceByName configuration = new DuccUimaReferenceByName(0, ddPath);
+ String ddNew = deployableGenerator.generateDd(configuration, System.getenv("DUCC_JOBID"), true);
if (ddNew != ddPath) {
UIMAFramework.getLogger(CLASS_NAME).log(Level.INFO, "Generated " + ddNew + " from " + ddPath);
}
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaProcessContainer.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaProcessContainer.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaProcessContainer.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jp/UimaProcessContainer.java Thu Jun 15 20:49:26 2017
@@ -20,6 +20,7 @@
package org.apache.uima.ducc.user.jp;
+import java.io.File;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
@@ -36,7 +37,6 @@ import org.apache.uima.analysis_engine.A
import org.apache.uima.cas.CAS;
import org.apache.uima.ducc.user.common.UimaUtils;
import org.apache.uima.ducc.user.jp.uima.UimaAnalysisEngineInstancePoolWithThreadAffinity;
-import org.apache.uima.impl.UimaVersion;
import org.apache.uima.resource.Resource;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.ResourceManager;
@@ -94,14 +94,15 @@ public class UimaProcessContainer extend
return threadAffinity;
}
+ // UIMA-5428 Generate the pieces-parts AE only if the JD-generated one is not accessible
private int configureAndGetScaleout(String[] args ) throws Exception {
analysisEngineDescriptor = ArgsParser.getArg("-aed", args);
+ if (analysisEngineDescriptor == null || !new File(analysisEngineDescriptor).canRead()) {
+ analysisEngineDescriptor = buildDeployable();
+ System.out.println("Created descriptor: "+analysisEngineDescriptor);
+ }
String threadCount = ArgsParser.getArg("-t", args); // Will be null if ducc.deploy.JpThreadCount is undefined
scaleout = threadCount==null ? 1 : Integer.valueOf(threadCount); // Default to 1
- String jobType = System.getProperty("ducc.deploy.JpType");
- if ( "uima".equals(jobType)) {
- System.out.println("UIMA Version:"+UimaVersion.getFullVersionString());
- }
return scaleout;
}
public byte[] getLastSerializedError() throws Exception {
@@ -114,9 +115,9 @@ public class UimaProcessContainer extend
}
public int doInitialize(Properties props, String[] args) throws Exception {
- buildDeployable();
return configureAndGetScaleout(args);
}
+
public void doDeploy() throws Exception {
System.out.println("....... UimaProcessContainer.doDeploy()");
Modified: uima/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java?rev=1798878&r1=1798877&r2=1798878&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java Thu Jun 15 20:49:26 2017
@@ -39,7 +39,7 @@ import org.apache.uima.ducc.user.dgen.De
import org.apache.uima.ducc.user.dgen.DuccUimaAggregate;
import org.apache.uima.ducc.user.dgen.DuccUimaAggregateComponent;
import org.apache.uima.ducc.user.dgen.IDuccGeneratorUimaAggregateComponent;
-import org.apache.uima.ducc.user.dgen.IDuccGeneratorUimaDeployableConfiguration;
+import org.apache.uima.ducc.user.dgen.IDuccGeneratorUimaAggregate;
import org.apache.uima.ducc.user.dgen.iface.DeployableGeneration;
import org.apache.uima.ducc.user.jd.JdUserCollectionReader;
import org.apache.uima.ducc.user.jd.JdUserMetaCas;
@@ -371,12 +371,8 @@ public class TestSuite {
}
}
- private IDuccGeneratorUimaDeployableConfiguration getIDuccUimaDeployableConfiguration() {
- String aeName = "name";
- String aeDescription = "description";
+ private IDuccGeneratorUimaAggregate getIDuccUimaDeployableConfiguration() {
int aeThreadCount = 1;
- String aeBrokerURL = "brokerURL";
- String aeEndpoint = "endpoint";
String aeFlowController = "flowController";
ArrayList<IDuccGeneratorUimaAggregateComponent> aeComponents = new ArrayList<IDuccGeneratorUimaAggregateComponent>();
URL url = this.getClass().getResource("/CR100.xml");
@@ -385,7 +381,7 @@ public class TestSuite {
List<String> aeOverrides = null;
DuccUimaAggregateComponent aeComponent = new DuccUimaAggregateComponent(aeDescriptor, aeOverrides);
aeComponents.add(aeComponent);
- IDuccGeneratorUimaDeployableConfiguration configuration = new DuccUimaAggregate(aeName, aeDescription, aeThreadCount, aeBrokerURL, aeEndpoint, aeFlowController, aeComponents);
+ IDuccGeneratorUimaAggregate configuration = new DuccUimaAggregate(aeThreadCount, aeFlowController, aeComponents);
return configuration;
}
@@ -416,9 +412,9 @@ public class TestSuite {
delete(working);
working.mkdir();
DeployableGenerator aeGenerator = new DeployableGenerator(working.getAbsolutePath());
- IDuccGeneratorUimaDeployableConfiguration configuration = getIDuccUimaDeployableConfiguration();
+ IDuccGeneratorUimaAggregate configuration = getIDuccUimaDeployableConfiguration();
String jobId = "12345";
- String ae = aeGenerator.generate(configuration, jobId);
+ String ae = aeGenerator.generateAe(configuration, jobId, false);
debug(ae);
//show(ae);
delete(working);
@@ -445,11 +441,8 @@ public class TestSuite {
String directory = working.getAbsolutePath();
String id = "99999";
- String aeName = "aeName";
- String aeDescription = "aeDescription";
+ Boolean aeCreateUniqueFilename = false;
Integer aeThreadCount = new Integer(11);
- String aeBrokerURL = "aeBrokerURL";
- String aeBrokerEndpoint = "aeBrokerEndpoint";
String aeFlowController = "aeFlowController";
String cmDescriptor = null;
List<String> cmOverrides = null;
@@ -460,18 +453,16 @@ public class TestSuite {
String ae = dg.generate(
directory,
- id, aeName,
- aeDescription,
+ id,
aeThreadCount,
- aeBrokerURL,
- aeBrokerEndpoint,
aeFlowController,
cmDescriptor,
cmOverrides,
aeDescriptor,
aeOverrides,
ccDescriptor,
- ccOverrides);
+ ccOverrides,
+ aeCreateUniqueFilename);
debug(ae);
//show(ae);