You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by bu...@apache.org on 2013/08/02 23:22:58 UTC
svn commit: r1509867 [1/2] - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/
uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/
uima-ducc-o...
Author: burn
Date: Fri Aug 2 21:22:58 2013
New Revision: 1509867
URL: http://svn.apache.org/r1509867
Log:
UIMA-2924 Add support for quoted values for JVM args and environment values
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobCancel.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationCancel.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationCancel.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationSubmit.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceCancel.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOne.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/pom.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java Fri Aug 2 21:22:58 2013
@@ -24,16 +24,19 @@ import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.lang.management.ManagementFactory;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.Parser;
import org.apache.commons.cli.PosixParser;
import org.apache.uima.ducc.common.IDucc;
import org.apache.uima.ducc.common.NodeIdentity;
@@ -55,11 +58,10 @@ public abstract class CliBase
private String myClassName = "N/A";
private boolean init_done = false;
protected String ducc_home;
- DuccProperties ducc_properties;
DuccEventHttpDispatcher dispatcher;
protected Options options;
- protected CommandLineParser parser;
+ protected Parser parser;
protected CommandLine commandLine;
protected long friendlyId = -1;
@@ -70,7 +72,7 @@ public abstract class CliBase
protected ArrayList<String> warnings = new ArrayList<String>();
protected ArrayList<String> messages = new ArrayList<String>();
- protected boolean debug = false;
+ protected boolean debug;
protected ConsoleListener console_listener = null;
protected String host_address = "N/A";
@@ -81,8 +83,10 @@ public abstract class CliBase
CountDownLatch waiter = null;
- protected Properties userSpecifiedProperties = new Properties();
- protected Properties fileSpecifiedProperties = new Properties();
+ protected Properties userSpecifiedProperties;
+
+ // Options added to the saved spec file that must be removed if used as a --specification option
+ private List<String> addedOptions = Arrays.asList(UiOption.SubmitPid.pname(), UiOption.User.pname());
/**
* All extenders must implement execute - this method does whatever processing on the input
@@ -93,7 +97,7 @@ public abstract class CliBase
*/
public abstract boolean execute() throws Exception;
- String getLogDirectory(String extension)
+ String getLogDirectory()
{
/*
* employ default log directory if not specified
@@ -115,9 +119,6 @@ public abstract class CliBase
}
}
}
- if ( extension != null ) {
- log_directory = log_directory + File.separator + extension;
- }
cli_props.setProperty(UiOption.LogDirectory.pname(), log_directory);
@@ -164,24 +165,13 @@ public abstract class CliBase
}
/*
- * resolve ${defaultBrokerURL} in service dependencies - must fail if resolution needed but can't resolve
+ * Check the syntax & if a service refers to itself -- place-holders already resolved
*/
- boolean resolve_service_dependencies(String endpoint)
+ boolean check_service_dependencies(String endpoint)
{
- String key_ja = UiOption.ProcessJvmArgs.pname();
- if ( cli_props.containsKey(UiOption.JvmArgs.pname()) ) {
- key_ja = UiOption.JvmArgs.pname();
- }
- String jvmargs = cli_props.getProperty(key_ja);
-
- Properties jvmprops = DuccUiUtilities.jvmArgsToProperties(jvmargs);
-
String deps = cli_props.getProperty(UiOption.ServiceDependency.pname());
try {
- deps = DuccUiUtilities.resolve_service_dependencies(endpoint, deps, jvmprops);
- if ( deps != null ) {
- cli_props.setProperty(UiOption.ServiceDependency.pname(), deps);
- }
+ DuccUiUtilities.check_service_dependencies(endpoint, deps);
return true;
} catch ( Throwable t ) {
message("ERROR:", t.toString());
@@ -208,7 +198,7 @@ public abstract class CliBase
}
}
- protected Options makeOptions(UiOption[] optlist, boolean strict)
+ protected Options makeOptions(UiOption[] optlist)
{
Options opts = new Options();
for ( UiOption opt : optlist ) {
@@ -216,7 +206,6 @@ public abstract class CliBase
Option o = new Option(opt.sname(), opt.pname(), (arg != null), opt.makeDesc());
o.setArgName(arg);
o.setOptionalArg(arg != null && arg.endsWith("(optional)"));
- o.setRequired(strict && opt.required());
if (opt.multiargs()) {
o.setArgs(Option.UNLIMITED_VALUES); // (Untested as we have no multiarg options)
}
@@ -256,48 +245,22 @@ public abstract class CliBase
return arglist.toArray(new String[arglist.size()]);
}
- private void enhanceProperties(CommandLine commandLine, boolean showdebug)
- {
- Option[] cliopts = commandLine.getOptions();
- for ( Option o : cliopts ) {
- if ( debug && showdebug ) {
- System.out.println("CLI Override: " + o.toString());
- }
- String k = o.getLongOpt().trim();
- String v = o.getValue();
- if ( v == null ) v = ""; // Boolean options have an empty value
- v = v.trim();
-
- cli_props.put(k, v);
- }
- }
-
-
/**
- * Use this init if you use the default log location and don't need a console callback.
+ * Standard init for all except the Service calls that are sent to the SM
*/
- protected synchronized void init(String myClassName, UiOption[] opts, String[] args, DuccProperties cli_props, String host_s, String port_s, String servlet)
- throws Exception
- {
- this.init(myClassName, opts, args, cli_props, host_s, port_s, servlet, null, null);
- }
- /**
- * Use this init if you you need a console callback and use the default log location.
- */
protected synchronized void init(String myClassName, UiOption[] opts, String[] args, DuccProperties cli_props,
- String host_s, String port_s, String servlet, IDuccCallback consoleCb)
- throws Exception
- {
- this.init(myClassName, opts, args, cli_props, host_s, port_s, servlet, consoleCb, null);
+ IDuccCallback consoleCb) throws Exception {
+ this.init (myClassName, opts, args, null, cli_props, consoleCb, "orchestrator");
}
- protected synchronized void init() {
- ducc_home = Utils.findDuccHome();
+ protected synchronized void init(String myClassName, UiOption[] opts, Properties props, DuccProperties cli_props,
+ IDuccCallback consoleCb) throws Exception {
+ this.init (myClassName, opts, null, props, cli_props, consoleCb, "orchestrator");
}
- protected synchronized void init(String myClassName, UiOption[] opts, String[] args, DuccProperties cli_props,
- String host_s, String port_s, String servlet, IDuccCallback consoleCb, String logExtension)
+ protected synchronized void init(String myClassName, UiOption[] opts, String[] args, Properties props,
+ DuccProperties cli_props, IDuccCallback consoleCb, String servlet)
throws Exception
{
if ( init_done ) return;
@@ -314,112 +277,149 @@ public abstract class CliBase
this.cli_props = cli_props;
parser = new PosixParser();
- // Set up for reverse lookup
-// for (UiOption opt : UiOption.values() ) {
-// reverseOptions.put(opt.pname(), opt);
-// }
-
-// options.addOption(OptionBuilder
-// .withArgName(DuccUiConstants.parm_driver_descriptor_CR)
-// .withDescription(makeDesc(DuccUiConstants.desc_driver_descriptor_CR,DuccUiConstants.exmp_driver_descriptor_CR)).hasArg()
-// .withLongOpt(DuccUiConstants.name_driver_descriptor_CR).create());
-
- // If given only a properties file convert to an array of strings for the parser
+ options = makeOptions(opts);
+ // If given only a properties file parse as if only have defaults
if (args == null) {
- args = mkArgs(cli_props);
- cli_props.clear();
- }
-
- // Initially don't check for required options as they may be in a specification file
- options = makeOptions(opts, false);
- commandLine = parser.parse(options, args);
-
- Option[] optionArray = commandLine.getOptions();
- for(Option option : optionArray) {
- String key = option.getLongOpt().trim();
- String value = option.getValue("").trim();
- userSpecifiedProperties.setProperty(key, value);
- }
-
- if (commandLine.hasOption(UiOption.Help.pname())) {
- usage(null);
- }
- if (commandLine.hasOption(UiOption.Debug.pname())) {
- debug = true;
+ commandLine = parser.parse(options, null, props);
+ } else {
+ fixupQuotedArgs(args);
+ commandLine = parser.parse(options, args);
}
- if(commandLine.getOptions().length == 0) {
+ if (commandLine.getOptions().length == 0 || commandLine.hasOption(UiOption.Help.pname())) {
usage(null);
}
+ debug = commandLine.hasOption(UiOption.Debug.pname());
// Load the specification file, if given on the command line. Note that registration
- // bypasses the somewhat redundant --specification kw so we check two options.
- String spec1 = UiOption.Specification.pname();
- String val = null;
- if ( commandLine.hasOption(spec1) ) {
- val = commandLine.getOptionValue(spec1);
- }
- String spec2 = UiOption.Register.pname();
- if ( commandLine.hasOption(spec2) ) {
- val = commandLine.getOptionValue(spec2);
- }
- if ( val != null ) {
- File file = new File(val);
- FileInputStream fis = new FileInputStream(file);
- cli_props.load(fis);
-
- String[] keyArray = cli_props.keySet().toArray(new String[0]);
- for(String key : keyArray) {
- String value = cli_props.getProperty(key);
- fileSpecifiedProperties.setProperty(key, value);
+ // bypasses the somewhat redundant --specification option so we check two options.
+ // Cannot have both as --specification && --register are never both valid.
+ String fname = null;
+ for (String spec : new String[] { UiOption.Specification.pname(), UiOption.Register.pname()}) {
+ fname = commandLine.getOptionValue(spec);
+ if (fname != null) break;
+ }
+ // If have a specification file re-parse using it for default values
+ if ( fname != null ) {
+ FileInputStream fis = new FileInputStream(new File(fname));
+ Properties defaults = new Properties();
+ defaults.load(fis);
+ fis.close();
+ sanitize(defaults, options); // Check for illegals as commons cli 1.2 thows a NPE !
+ // If invoked with overriding properties add to or replace the defaults
+ if (props != null) {
+ defaults.putAll(props);
}
-
- // Loop through options and enhance / override things from cl options
- enhanceProperties(commandLine, true);
-
- // Now a trick - we'll rebuild the command line with the props as well as the cli args
- // and reparse strictly.
- args = mkArgs(cli_props);
- cli_props.clear();
- }
-
- // Even if no specification file provided, re-parse and check for required options
- options = makeOptions(opts, true);
- commandLine = parser.parse(options, args);
- enhanceProperties(commandLine, false);
-
- String propsfile = ducc_home + "/resources/ducc.properties";
- ducc_properties = new DuccProperties();
- ducc_properties.load(propsfile);
- cli_props.setProperty(UiOption.SubmitPid.pname(), ManagementFactory.getRuntimeMXBean().getName()); // my pid
-
- String host = ducc_properties.getStringProperty(host_s);
- String port = ducc_properties.getStringProperty(port_s);
-
- if ( host == null ) {
- throw new IllegalStateException(host_s + " is not set in ducc.properties");
+ commandLine = parser.parse(options, args, defaults);
}
+
+ // Copy options into cli_props
+ setOptions(opts);
- if ( port == null ) {
- throw new IllegalStateException(port_s + " is not set in ducc.properties");
- }
-
- String targetUrl = "http://"+ host + ":" + port + "/" + servlet;
- dispatcher = new DuccEventHttpDispatcher(targetUrl);
+ // Save a copy of the user-specified ones by cloning the underlying properties
+ userSpecifiedProperties = (Properties)((Properties)cli_props).clone();
+
+ cli_props.setProperty(UiOption.SubmitPid.pname(), ManagementFactory.getRuntimeMXBean().getName());
- if ( getLogDirectory(logExtension) == null ) {
+ if ( getLogDirectory() == null ) {
throw new IllegalArgumentException("Cannot access log directory.");
}
setWorkingDirectory();
setUser();
+ //TODO - shouldn't environment fixups be done here for all requests that may use it??
+
NodeIdentity ni = new NodeIdentity();
- this.host_address = ni.getIp();
+ host_address = ni.getIp();
initConsoleListener();
+ String targetUrl = DuccUiUtilities.dispatchUrl(servlet);
+ dispatcher = new DuccEventHttpDispatcher(targetUrl);
+
init_done = true;
}
+ /*
+ * Save options as properties after resolving any ${..} placeholders
+ * Also check that all required ones provided
+ */
+ void setOptions(UiOption[] opts) throws Exception {
+ for (Option opt : commandLine.getOptions()) {
+ String val = opt.getValue();
+ if (val == null) {
+ val = "true"; // Treat no-arg options as booleans ... apache.commons.cli expects this
+ } else {
+ if (val.contains("${")) {
+ val = resolvePlaceholders(val);
+ }
+ }
+ cli_props.put(opt.getLongOpt(), val);
+ if (debug) System.out.println("CLI set " + opt.getLongOpt() + " = " + val);
+ }
+
+ for (UiOption opt : opts) {
+ if (opt.required() && !cli_props.containsKey(opt.pname())) {
+ throw new Exception("Missing required option: " + opt.pname());
+ }
+ }
+ }
+
+ /*
+ * Clean up the properties in a specification file
+ * Remove any added by the CLI that the parse would call illegal
+ * Check for invalid options as Commons CLI 1.2 throws a NPE
+ * Correct booleans by treating empty as "true" and removing anything
+ * other than 'true' or 'yes' or '1' (CLI 1.2 mishandles others)
+ */
+
+ private void sanitize(Properties props, Options opts) {
+ for (String key : props.stringPropertyNames()) {
+ if (addedOptions.contains(key)) {
+ props.remove(key);
+ } else {
+ Option opt = options.getOption(key);
+ if (opt == null) {
+ throw new IllegalArgumentException("Invalid option " + key + " in specification file");
+ }
+ if (!opt.hasArg()) {
+ String val = props.getProperty(key);
+ if (val.length() == 0) {
+ props.setProperty(key, "true");
+ } else if (!val.equalsIgnoreCase("true") &&
+ !val.equalsIgnoreCase("yes") &&
+ !val.equals("1")) {
+ message("WARN: Ignoring illegal value: ", key, "=", val);
+ props.remove(key);
+ }
+ }
+ }
+ }
+ }
+
+ /*
+ * Resolve any ${..} placeholders against user's system properties and environment
+ */
+ private String resolvePlaceholders(String contents) {
+ // Placeholders syntax ${<placeholder>}
+ Pattern pattern = Pattern.compile("\\$\\{(.*?)\\}"); // Stops on first '}'
+ Matcher matcher = pattern.matcher(contents);
+
+ StringBuffer sb = new StringBuffer();
+ while (matcher.find()) {
+ final String key = matcher.group(1);
+ String value = System.getProperty(key);
+ if (value == null) {
+ value = System.getenv(key);
+ if (value == null) {
+ throw new IllegalArgumentException("Missing value for placeholder '" + key + "' in: " + contents);
+ }
+ }
+ matcher.appendReplacement(sb, value);
+ }
+ matcher.appendTail(sb);
+ return sb.toString();
+ }
+
void saveSpec(String name, DuccProperties props)
throws Exception
{
@@ -432,12 +432,9 @@ public abstract class CliBase
throw new IllegalStateException("saveSpec: Cannot create log directory: " + f.toString());
}
+ // Save the specification (but exclude the 'signature' entry)
String comments = null;
- FileOutputStream fos = null;
- OutputStreamWriter out = null;
- fos = new FileOutputStream(fileName);
- out = new OutputStreamWriter(fos);
-
+ OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(fileName));
String key = UiOption.Signature.pname();
if ( props.containsKey(key) ) {
Object value = props.remove(key);
@@ -446,27 +443,13 @@ public abstract class CliBase
} else {
props.store(out, comments);
}
-
out.close();
- fos.close();
-
- /////
+ // Also save just the values the user provided
fileName = directory + File.separator + DuccUiConstants.user_specified_properties;
- fos = new FileOutputStream(fileName);
- out = new OutputStreamWriter(fos);
+ out = new OutputStreamWriter(new FileOutputStream(fileName));
userSpecifiedProperties.store(out, comments);
out.close();
- fos.close();
-
- /////
-
- fileName = directory + File.separator + DuccUiConstants.file_specified_properties;
- fos = new FileOutputStream(fileName);
- out = new OutputStreamWriter(fos);
- fileSpecifiedProperties.store(out, comments);
- out.close();
- fos.close();
}
/**
@@ -557,27 +540,28 @@ public abstract class CliBase
* Return internal API debug status.
* @return True if the API debugging flag is set; false otherwise.
*/
- public boolean isDebug()
+/* public boolean isDebug()
{
return debug;
- }
+ }*/
/**
* Set the internal API debug flag.
* @param val Set to true to enable debugging, and false to disable it.
*/
- public void setDebug(boolean val)
+/* public void setDebug(boolean val)
{
this.debug = val;
}
-
+*/
// nobody seems to use this
// public String getHostAddress()
// {
// return host_address;
// }
- public boolean hasProperty(String key)
+/* Also unused?
+ * public boolean hasProperty(String key)
{
return cli_props.containsKey(key);
}
@@ -585,7 +569,7 @@ public abstract class CliBase
public String getProperty(String key)
{
return (String) cli_props.getProperty(key);
- }
+ }*/
protected IDuccCallback getCallback()
{
@@ -805,4 +789,20 @@ public abstract class CliBase
return false;
}
+ /*
+ * Since apache-commons-cli 1.2 wrongly removes initial or final quotes, add extra one(s)
+ * i.e. an --environment setting of FOO="a b" becomes FOO="a b""
+ * What about a lonely " ... both starts & ends so would become => """
+ */
+ private String[] fixupQuotedArgs(String[] args) {
+ for (int i = 0; i < args.length; ++i) {
+ if (args[i].charAt(0) == '"') {
+ args[i] = "\"" + args[i];
+ }
+ if (args[i].endsWith("\"")) {
+ args[i] = args[i] + "\"";
+ }
+ }
+ return args;
+ }
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java Fri Aug 2 21:22:58 2013
@@ -63,7 +63,7 @@ class ConsoleListener
NodeIdentity ni = new NodeIdentity();
this.console_host_address = ni.getIp();
- debug = submit.isDebug();
+ debug = submit.debug; //isDebug();
}
String getConsoleHostAddress()
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobCancel.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobCancel.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobCancel.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobCancel.java Fri Aug 2 21:22:58 2013
@@ -34,8 +34,6 @@ public class DuccJobCancel
{
JobRequestProperties jobRequestProperties = new JobRequestProperties();
- static String or_port = "ducc.orchestrator.http.port";
- static String or_host = "ducc.orchestrator.http.node";
long canceledPid = -1;
String responseMessage = null;
@@ -59,7 +57,7 @@ public class DuccJobCancel
public DuccJobCancel(String [] args)
throws Exception
{
- init(this.getClass().getName(), opts, args, jobRequestProperties, or_host, or_port, "or", null);
+ init (this.getClass().getName(), opts, args, jobRequestProperties, null);
}
/**
@@ -70,7 +68,7 @@ public class DuccJobCancel
throws Exception
{
String[] arg_array = args.toArray(new String[args.size()]);
- init(this.getClass().getName(), opts, arg_array, jobRequestProperties, or_host, or_port, "or", null);
+ init(this.getClass().getName(), opts, arg_array, jobRequestProperties, null);
}
/**
@@ -80,11 +78,7 @@ public class DuccJobCancel
public DuccJobCancel(Properties props)
throws Exception
{
- for ( Object k : props.keySet() ) {
- Object v = props.get(k);
- jobRequestProperties.put(k, v);
- }
- init(this.getClass().getName(), opts, null, jobRequestProperties, or_host, or_port, "or", null, null);
+ init (this.getClass().getName(), opts, props, jobRequestProperties, null);
}
/**
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java Fri Aug 2 21:22:58 2013
@@ -39,12 +39,6 @@ public class DuccJobSubmit
extends CliBase
{
private JobRequestProperties jobRequestProperties = new JobRequestProperties();
- static String or_port = "ducc.orchestrator.http.port";
- static String or_host = "ducc.orchestrator.http.node";
-
- // public DuccJobSubmit(IDuccMessageProcessor duccMessageProcessor) {
-// this.duccMessageProcessor = duccMessageProcessor;
-// }
static UiOption[] opts_release = new UiOption[] {
UiOption.Help,
@@ -218,12 +212,10 @@ public class DuccJobSubmit
public DuccJobSubmit(String[] args, IDuccCallback consoleCb)
throws Exception
{
- init();
if(DuccUiUtilities.isSupportedBeta()) {
opts = opts_beta;
}
- init(this.getClass().getName(), opts, args, jobRequestProperties, or_host, or_port, "or", consoleCb, null);
- enrich_parameters_with_defaults(this, jobRequestProperties);
+ init (this.getClass().getName(), opts, args, jobRequestProperties, consoleCb);
if(isAllInOne()) {
allInOneLauncher = new AllInOneLauncher(args, consoleCb);
}
@@ -241,23 +233,17 @@ public class DuccJobSubmit
public DuccJobSubmit(Properties props, IDuccCallback consoleCb)
throws Exception
{
- for ( Object k : props.keySet() ) {
- Object v = props.get(k);
- jobRequestProperties.put(k, v);
- }
- init();
- if(DuccUiUtilities.isSupportedBeta()) {
+ if (DuccUiUtilities.isSupportedBeta()) {
opts = opts_beta;
}
- init(this.getClass().getName(), opts, null, jobRequestProperties, or_host, or_port, "or", consoleCb, null);
- enrich_parameters_with_defaults(this, jobRequestProperties);
+ init (this.getClass().getName(), opts, props, jobRequestProperties, consoleCb);
if(isAllInOne()) {
String[] args = mkArgs(props);
allInOneLauncher = new AllInOneLauncher(args, consoleCb);
}
}
- protected void enrich_parameters_with_defaults(CliBase base, Properties props)
+ protected void enrich_parameters_with_defaults(Properties props)
throws Exception
{
String pname = UiOption.SchedulingClass.pname();
@@ -267,7 +253,7 @@ public class DuccJobSubmit
if(scheduling_class != null) {
props.put(pname, scheduling_class);
String text = pname+"="+scheduling_class+" [default]";
- base.message(text);
+ message(text);
}
else {
throw new MissingArgumentException(pname);
@@ -469,7 +455,7 @@ public class DuccJobSubmit
if(allInOneLauncher != null) {
return allInOneLauncher.execute();
}
-
+ enrich_parameters_with_defaults(jobRequestProperties);
try {
enrich_parameters_for_debug(jobRequestProperties);
} catch (Exception e1) {
@@ -541,7 +527,7 @@ public class DuccJobSubmit
}
/*
- * set DUCC_LD_LIBRARY_PATH in driver, process environment
+ * Augment the environment(s) with DUCC_LD_LIBRARY_PATH and any propagetd values
*/
boolean ev0 = jobRequestProperties.containsKey(UiOption.Environment.pname());
boolean evd = jobRequestProperties.containsKey(UiOption.DriverEnvironment.pname());
@@ -597,7 +583,7 @@ public class DuccJobSubmit
/*
* resolve ${defaultBrokerURL} in service dependencies - must fail if resolution needed but can't resolve
*/
- if ( ! resolve_service_dependencies(null) ) {
+ if ( ! check_service_dependencies(null) ) {
return false;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationCancel.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationCancel.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationCancel.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationCancel.java Fri Aug 2 21:22:58 2013
@@ -32,8 +32,6 @@ import org.apache.uima.ducc.transport.ev
public class DuccManagedReservationCancel extends CliBase {
JobRequestProperties requestProperties = new JobRequestProperties();
- static String or_port = "ducc.orchestrator.http.port";
- static String or_host = "ducc.orchestrator.http.node";
long canceledPid = -1;
String responseMessage = null;
@@ -55,7 +53,7 @@ public class DuccManagedReservationCance
public DuccManagedReservationCancel(String [] args)
throws Exception
{
- init(this.getClass().getName(), opts, args, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, args, requestProperties, null);
}
/**
@@ -66,7 +64,7 @@ public class DuccManagedReservationCance
throws Exception
{
String[] arg_array = args.toArray(new String[args.size()]);
- init(this.getClass().getName(), opts, arg_array, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, arg_array, requestProperties, null);
}
/**
@@ -76,11 +74,7 @@ public class DuccManagedReservationCance
public DuccManagedReservationCancel(Properties props)
throws Exception
{
- for ( Object k : props.keySet() ) {
- Object v = props.get(k);
- requestProperties.put(k, v);
- }
- init(this.getClass().getName(), opts, null, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, props, requestProperties, null);
}
/**
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java Fri Aug 2 21:22:58 2013
@@ -37,9 +37,6 @@ public class DuccManagedReservationSubmi
{
private static String dt = "Managed Reservation";
- private static String or_port = "ducc.orchestrator.http.port";
- private static String or_host = "ducc.orchestrator.http.node";
-
private ServiceRequestProperties serviceRequestProperties;
private UiOption[] opts_release = new UiOption[] {
@@ -125,11 +122,10 @@ public class DuccManagedReservationSubmi
throws Exception
{
serviceRequestProperties = new ServiceRequestProperties();
- init();
if(DuccUiUtilities.isSupportedBeta()) {
opts = opts_beta;
}
- init(this.getClass().getName(), opts, args, serviceRequestProperties, or_host, or_port, "or", consoleCb);
+ init (this.getClass().getName(), opts, args, serviceRequestProperties, consoleCb);
}
/**
@@ -146,11 +142,10 @@ public class DuccManagedReservationSubmi
{
String[] arg_array = args.toArray(new String[args.size()]);
serviceRequestProperties = new ServiceRequestProperties();
- init();
if(DuccUiUtilities.isSupportedBeta()) {
opts = opts_beta;
}
- init(this.getClass().getName(), opts, arg_array, serviceRequestProperties, or_host, or_port, "or", consoleCb);
+ init (this.getClass().getName(), opts, arg_array, serviceRequestProperties, consoleCb);
}
/**
@@ -166,17 +161,10 @@ public class DuccManagedReservationSubmi
throws Exception
{
serviceRequestProperties = new ServiceRequestProperties();
- // TODO - can we get OR to just use properties and not these specialized classes?
- // Until then, we need to pass in the right kind of props, sigh.
- for ( Object k : props.keySet() ) {
- Object v = props.get(k);
- serviceRequestProperties.put(k, v);
- }
- init();
if(DuccUiUtilities.isSupportedBeta()) {
opts = opts_beta;
}
- init(this.getClass().getName(), opts, null, serviceRequestProperties, or_host, or_port, "or", consoleCb);
+ init (this.getClass().getName(), opts, props, serviceRequestProperties, consoleCb);
}
@@ -189,11 +177,6 @@ public class DuccManagedReservationSubmi
public boolean execute() throws Exception
{
- // These must be enforced for "ducclets"
- serviceRequestProperties.setProperty(UiOption.ProcessThreadCount.pname(), "1");
- serviceRequestProperties.setProperty(UiOption.ProcessDeploymentsMax.pname(), "1");
- serviceRequestProperties.put(UiOption.ServiceTypeOther.pname(), "");
-
/*
* set DUCC_LD_LIBRARY_PATH in process environment
*/
@@ -220,6 +203,12 @@ public class DuccManagedReservationSubmi
}
}
+ // Create a copy to be saved later without these 3 "ducclet" properties required by DUCC
+ ServiceRequestProperties serviceProperties = (ServiceRequestProperties)serviceRequestProperties.clone();
+ serviceRequestProperties.setProperty(UiOption.ProcessThreadCount.pname(), "1");
+ serviceRequestProperties.setProperty(UiOption.ProcessDeploymentsMax.pname(), "1");
+ serviceRequestProperties.setProperty(UiOption.ServiceTypeOther.pname(), "");
+
SubmitServiceDuccEvent ev = new SubmitServiceDuccEvent(serviceRequestProperties);
SubmitServiceReplyDuccEvent reply = null;
@@ -260,7 +249,7 @@ public class DuccManagedReservationSubmi
if ( friendlyId < 0 ) {
retval = false;
} else {
- saveSpec(DuccUiConstants.managed_reservation_properties, serviceRequestProperties);
+ saveSpec(DuccUiConstants.managed_reservation_properties, serviceProperties);
startMonitors(true, DuccContext.ManagedReservation); // starts conditionally, based on job spec and console listener present
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationCancel.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationCancel.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationCancel.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationCancel.java Fri Aug 2 21:22:58 2013
@@ -34,8 +34,6 @@ public class DuccReservationCancel
{
ReservationRequestProperties requestProperties = new ReservationRequestProperties();
- static String or_port = "ducc.orchestrator.http.port";
- static String or_host = "ducc.orchestrator.http.node";
long canceledPid = -1;
String responseMessage = null;
@@ -56,7 +54,7 @@ public class DuccReservationCancel
public DuccReservationCancel(String [] args)
throws Exception
{
- init(this.getClass().getName(), opts, args, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, args, requestProperties, null);
}
/**
@@ -67,7 +65,7 @@ public class DuccReservationCancel
throws Exception
{
String[] arg_array = args.toArray(new String[args.size()]);
- init(this.getClass().getName(), opts, arg_array, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, arg_array, requestProperties, null);
}
/**
@@ -77,11 +75,7 @@ public class DuccReservationCancel
public DuccReservationCancel(Properties props)
throws Exception
{
- for ( Object k : props.keySet() ) {
- Object v = props.get(k);
- requestProperties.put(k, v);
- }
- init(this.getClass().getName(), opts, null, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, props, requestProperties, null);
}
/**
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationSubmit.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationSubmit.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationSubmit.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationSubmit.java Fri Aug 2 21:22:58 2013
@@ -33,9 +33,6 @@ import org.apache.uima.ducc.transport.ev
public class DuccReservationSubmit
extends CliBase
{
- static String or_port = "ducc.orchestrator.http.port";
- static String or_host = "ducc.orchestrator.http.node";
-
ReservationRequestProperties requestProperties = new ReservationRequestProperties();
private String nodeList = "";
@@ -47,7 +44,7 @@ public class DuccReservationSubmit
public DuccReservationSubmit(String[] args)
throws Exception
{
- init(this.getClass().getName(), opts, args, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, args, requestProperties, null);
}
/**
@@ -58,7 +55,7 @@ public class DuccReservationSubmit
throws Exception
{
String[] arg_array = args.toArray(new String[args.size()]);
- init(this.getClass().getName(), opts, arg_array, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, arg_array, requestProperties, null);
}
/**
@@ -68,11 +65,7 @@ public class DuccReservationSubmit
public DuccReservationSubmit(Properties props)
throws Exception
{
- for ( Object k : props.keySet() ) {
- Object v = props.get(k);
- requestProperties.put(k, v);
- }
- init(this.getClass().getName(), opts, null, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, props, requestProperties, null);
}
UiOption[] opts = new UiOption[] {
@@ -86,33 +79,6 @@ public class DuccReservationSubmit
UiOption.ReservationMemorySize,
};
-// @SuppressWarnings("static-access")
-// private void addOptions(Options options) {
-// // // options.addOption(OptionBuilder
-// // // .withDescription(DuccUiConstants.desc_help).hasArg(false)
-// // // .withLongOpt(DuccUiConstants.name_help).create());
-// // options.addOption(OptionBuilder
-// // .withArgName(DuccUiConstants.parm_description)
-// // .withDescription(makeDesc(DuccUiConstants.desc_description,DuccUiConstants.exmp_description)).hasArg()
-// // .withLongOpt(DuccUiConstants.name_description).create());
-// options.addOption(OptionBuilder
-// .withArgName(DuccUiConstants.parm_reservation_scheduling_class)
-// .withDescription(makeDesc(DuccUiConstants.desc_reservation_scheduling_class,DuccUiConstants.exmp_reservation_scheduling_class)).hasArg()
-// .withLongOpt(DuccUiConstants.name_reservation_scheduling_class).create());
-// options.addOption(OptionBuilder
-// .withArgName(DuccUiConstants.parm_number_of_instances)
-// .withDescription(makeDesc(DuccUiConstants.desc_number_of_instances,DuccUiConstants.exmp_number_of_instances)).hasArg()
-// .withLongOpt(DuccUiConstants.name_number_of_instances).create());
-// options.addOption(OptionBuilder
-// .withArgName(DuccUiConstants.parm_instance_memory_size)
-// .withDescription(makeDesc(DuccUiConstants.desc_instance_memory_size,DuccUiConstants.exmp_instance_memory_size)).hasArg()
-// .withLongOpt(DuccUiConstants.name_instance_memory_size).create());
-// options.addOption(OptionBuilder
-// .withArgName(DuccUiConstants.parm_specification)
-// .withDescription(DuccUiConstants.desc_specification).hasArg()
-// .withLongOpt(DuccUiConstants.name_specification).create());
-// }
-
/**
* Execute collects the parameters for the reservation and sends them to the DUCC Orchestrator
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java Fri Aug 2 21:22:58 2013
@@ -20,12 +20,12 @@ package org.apache.uima.ducc.cli;
import java.util.ArrayList;
import java.util.List;
-import java.util.Properties;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.uima.ducc.common.Pair;
import org.apache.uima.ducc.common.utils.DuccProperties;
+import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
import org.apache.uima.ducc.transport.event.ServiceModifyEvent;
import org.apache.uima.ducc.transport.event.ServiceQueryEvent;
import org.apache.uima.ducc.transport.event.ServiceRegisterEvent;
@@ -46,8 +46,6 @@ public class DuccServiceApi
extends CliBase
{
- String sm_port = "ducc.sm.http.port";
- String sm_host = "ducc.sm.http.node";
String endpoint = null;
IDuccCallback callback = null;
@@ -238,7 +236,7 @@ public class DuccServiceApi
private void setLinger()
{
- String default_linger = ducc_properties.getStringProperty("ducc.sm.default.linger", "5000");
+ String default_linger = DuccPropertiesResolver.get("ducc.sm.default.linger", "5000");
String linger = cli_props.getStringProperty(UiOption.ServiceLinger.pname(), default_linger);
try {
@SuppressWarnings("unused")
@@ -248,17 +246,8 @@ public class DuccServiceApi
}
}
- String extractEndpoint(Properties jvmargs)
+ String extractEndpoint(String jvmargs)
{
- // If classpath is not specified, pick it up from the submitter's environment
- String key_cp = UiOption.ProcessClasspath.pname();
- if ( cli_props.containsKey(UiOption.Classpath.pname()) ) {
- key_cp = UiOption.Classpath.pname();
- }
- String classpath = cli_props.getStringProperty(key_cp, System.getProperty("java.class.path"));
- cli_props.setProperty(key_cp, classpath);
-
- // No endpoint, resolve from the DD.
String dd = cli_props.getStringProperty(UiOption.ProcessDD.pname()); // will throw if can't find the prop
String working_dir = cli_props.getStringProperty(UiOption.WorkingDirectory.pname());
endpoint = DuccUiUtilities.getEndpoint(working_dir, dd, jvmargs);
@@ -278,26 +267,13 @@ public class DuccServiceApi
throws Exception
{
DuccProperties dp = new DuccProperties();
- init();
if(DuccUiUtilities.isSupportedBeta()) {
registration_options = registration_options_beta;
}
- init(this.getClass().getName(), registration_options, args, dp, sm_host, sm_port, "sm", callback, null);
+ init (this.getClass().getName(), registration_options, args, null, dp, callback, "sm");
// Note: dp & cli_props are identical ... use only the class variable here for consistency
- //
- // Now: get jvm args and resolve placeholders, in particular, the broker url
- //
- String key_ja = UiOption.ProcessJvmArgs.pname();
- if ( cli_props.containsKey(UiOption.JvmArgs.pname()) ) {
- key_ja = UiOption.JvmArgs.pname();
- }
- String jvmarg_string = cli_props.getProperty(key_ja);
- Properties jvmargs = DuccUiUtilities.jvmArgsToProperties(jvmarg_string);
- // Presumably only the endpoint and service dependencies can have place holders
- DuccUiUtilities.resolvePropertiesPlaceholders(cli_props, jvmargs);
-
/*
* Fixup the environment: rename LD_LIBRARY_PATH & add any standard ones
*/
@@ -330,6 +306,7 @@ public class DuccServiceApi
message("WARN", "--process_executable is ignored for UIMA-AS services");
}
// Infer the classpath (DuccProperties will return the default if the value isn't found.)
+ // Note: only used for UIMA-AS services
String key_cp = UiOption.ProcessClasspath.pname();
if ( cli_props.containsKey(UiOption.Classpath.pname()) ) {
key_cp = UiOption.Classpath.pname();
@@ -340,13 +317,20 @@ public class DuccServiceApi
// Given ep must match inferred ep. Use case: an application is wrapping DuccServiceApi and has to construct
// the endpoint as well. The app passes it in and we insure that the constructed endpoint matches the one
// we extract from the DD - the job will fail otherwise, so we catch this early.
- String inferred_endpoint = extractEndpoint(jvmargs);
+ //
+ String key_ja = UiOption.ProcessJvmArgs.pname();
+ if ( cli_props.containsKey(UiOption.JvmArgs.pname()) ) {
+ key_ja = UiOption.JvmArgs.pname();
+ }
+ String jvmarg_string = cli_props.getProperty(key_ja);
+
+ String inferred_endpoint = extractEndpoint(jvmarg_string);
if (endpoint == null) {
endpoint = inferred_endpoint;
} else if ( !inferred_endpoint.equals(endpoint) ) {
- throw new IllegalArgumentException("Endpoint from --service_request_endpoint does not match endpoint ectracted from UIMA DD"
- + "\n--service_request_endpoint: " + endpoint
- + "\nextracted: : " + inferred_endpoint );
+ throw new IllegalArgumentException("Specified endpoint does not match endpoint extracted from UIMA DD"
+ + "\n --service_request_endpoint: " + endpoint
+ + "\n extracted: " + inferred_endpoint );
}
} else if (endpoint.startsWith(ServiceType.Custom.decode())) {
@@ -368,7 +352,7 @@ public class DuccServiceApi
}
// work out stuff I'm dependent upon
- if ( !resolve_service_dependencies(endpoint) ) {
+ if ( !check_service_dependencies(endpoint) ) {
throw new IllegalArgumentException("Invalid service dependencies");
}
int instances = cli_props.getIntProperty(UiOption.Instances.pname(), 1);
@@ -402,7 +386,8 @@ public class DuccServiceApi
throws Exception
{
DuccProperties dp = new DuccProperties();
- init(this.getClass().getName(), unregister_options, args, dp, sm_host, sm_port, "sm", callback, null);
+ init(this.getClass().getName(), unregister_options, args, null, dp, callback, "sm");
+
Pair<Integer, String> id = getId(UiOption.Unregister);
String user = dp.getProperty(UiOption.User.pname());
@@ -428,7 +413,7 @@ public class DuccServiceApi
throws Exception
{
DuccProperties dp = new DuccProperties();
- init(this.getClass().getName(), start_options, args, dp, sm_host, sm_port, "sm", callback, null);
+ init(this.getClass().getName(), start_options, args, null, dp, callback, "sm");
Pair<Integer, String> id = getId(UiOption.Start);
String user = dp.getProperty(UiOption.User.pname());
@@ -460,7 +445,7 @@ public class DuccServiceApi
throws Exception
{
DuccProperties dp = new DuccProperties();
- init(this.getClass().getName(), stop_options, args, dp, sm_host, sm_port, "sm", callback, null);
+ init(this.getClass().getName(), stop_options, args, null, dp, callback, "sm");
Pair<Integer, String> id = getId(UiOption.Stop);
String user = dp.getProperty(UiOption.User.pname());
@@ -492,7 +477,7 @@ public class DuccServiceApi
throws Exception
{
DuccProperties dp = new DuccProperties();
- init(this.getClass().getName(), modify_options, args, dp, sm_host, sm_port, "sm", callback, null);
+ init (this.getClass().getName(), modify_options, args, null, dp, callback, "sm");
Pair<Integer, String> id = getId(UiOption.Modify);
String user = dp.getProperty(UiOption.User.pname());
@@ -524,7 +509,7 @@ public class DuccServiceApi
throws Exception
{
DuccProperties dp = new DuccProperties();
- init(this.getClass().getName(), query_options, args, dp, sm_host, sm_port, "sm", callback, null);
+ init(this.getClass().getName(), query_options, args, null, dp, callback, "sm");
Pair<Integer, String> id = null;
String sid = cli_props.getProperty(UiOption.Query.pname()).trim();
@@ -554,27 +539,27 @@ public class DuccServiceApi
formatter.setWidth(IUiOptions.help_width);
System.out.println("------------- Register Options ------------------");
- options = makeOptions(registration_options, true);
+ options = makeOptions(registration_options);
formatter.printHelp(this.getClass().getName(), options);
System.out.println("\n\n------------- Unregister Options ------------------");
- options = makeOptions(unregister_options, true);
+ options = makeOptions(unregister_options);
formatter.printHelp(this.getClass().getName(), options);
System.out.println("\n\n------------- Start Options ------------------");
- options = makeOptions(start_options, true);
+ options = makeOptions(start_options);
formatter.printHelp(this.getClass().getName(), options);
System.out.println("\n\n------------- Stop Options ------------------");
- options = makeOptions(stop_options, true);
+ options = makeOptions(stop_options);
formatter.printHelp(this.getClass().getName(), options);
System.out.println("\n\n------------- Modify Options ------------------");
- options = makeOptions(modify_options, true);
+ options = makeOptions(modify_options);
formatter.printHelp(this.getClass().getName(), options);
System.out.println("\n\n------------- Query Options ------------------");
- options = makeOptions(query_options, true);
+ options = makeOptions(query_options);
formatter.printHelp(this.getClass().getName(), options);
System.exit(1);
@@ -757,7 +742,8 @@ public class DuccServiceApi
System.exit(1);
}
} catch (Exception e) {
- System.out.println("Service call failed: " + e.toString());
+ System.out.println("Service call failed: " + e);
+ //e.printStackTrace();
System.exit(1);
}
System.exit(rc ? 0 : 1);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceCancel.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceCancel.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceCancel.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceCancel.java Fri Aug 2 21:22:58 2013
@@ -35,8 +35,6 @@ public class DuccServiceCancel
{
JobRequestProperties requestProperties = new JobRequestProperties();
- static String or_port = "ducc.orchestrator.http.port";
- static String or_host = "ducc.orchestrator.http.node";
long canceledPid = -1;
String responseMessage = null;
@@ -58,7 +56,7 @@ public class DuccServiceCancel
public DuccServiceCancel(String [] args)
throws Exception
{
- init(this.getClass().getName(), opts, args, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, args, requestProperties, null);
}
/**
@@ -69,7 +67,7 @@ public class DuccServiceCancel
throws Exception
{
String[] arg_array = args.toArray(new String[args.size()]);
- init(this.getClass().getName(), opts, arg_array, requestProperties, or_host, or_port, "or");
+ init(this.getClass().getName(), opts, arg_array, requestProperties, null);
}
/**
@@ -79,11 +77,7 @@ public class DuccServiceCancel
public DuccServiceCancel(Properties props)
throws Exception
{
- for ( Object k : props.keySet() ) {
- Object v = props.get(k);
- requestProperties.put(k, v);
- }
- init(this.getClass().getName(), opts, null, requestProperties, or_host, or_port, "or");
+ init (this.getClass().getName(), opts, props, requestProperties, null);
}
/**
@@ -96,11 +90,6 @@ public class DuccServiceCancel
return responseMessage;
}
- // public boolean isService()
-// {
-// return true;
-// }
-
/**
* Execute collects the parameters for job cancelation and sends them to the DUCC Orchestrator
* to effect the cancelation.
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java Fri Aug 2 21:22:58 2013
@@ -41,8 +41,6 @@ public class DuccServiceSubmit
//private String jvmarg_string = null;
//private Properties jvmargs = null;
ServiceRequestProperties requestProperties = new ServiceRequestProperties();
- static String or_port = "ducc.orchestrator.http.port";
- static String or_host = "ducc.orchestrator.http.node";
UiOption[] opts_release = {
UiOption.Help,
@@ -114,11 +112,10 @@ public class DuccServiceSubmit
public DuccServiceSubmit(String[] args)
throws Exception
{
- init();
if(DuccUiUtilities.isSupportedBeta()) {
opts = opts_beta;
}
- init(this.getClass().getName(), opts, args, requestProperties, or_host, or_port, "or", null, null);
+ init(this.getClass().getName(), opts, args, requestProperties, null);
}
/**
@@ -138,15 +135,10 @@ public class DuccServiceSubmit
public DuccServiceSubmit(Properties props)
throws Exception
{
- for ( Object k : props.keySet() ) {
- Object v = props.get(k);
- requestProperties.put(k, v);
- }
- init();
if(DuccUiUtilities.isSupportedBeta()) {
opts = opts_beta;
}
- init(this.getClass().getName(), opts, null, requestProperties, or_host, or_port, "or", null, null);
+ init (this.getClass().getName(), opts, props, requestProperties, null);
}
/**
@@ -162,8 +154,7 @@ public class DuccServiceSubmit
if ( cli_props.containsKey(UiOption.JvmArgs.pname()) ) {
key_ja = UiOption.JvmArgs.pname();
}
- String jvmarg_string = (String) requestProperties.get(key_ja);
- Properties jvmargs = DuccUiUtilities.jvmArgsToProperties(jvmarg_string);
+ String jvmarg_string = requestProperties.getProperty(key_ja);
//
// Need to check if the mutually exclusive UIMA-AS DD and the Custom executable are specified
@@ -188,7 +179,7 @@ public class DuccServiceSubmit
try {
String dd = (String) requestProperties.get(UiOption.ProcessDD.pname());
String wd = (String) requestProperties.get(UiOption.WorkingDirectory.pname());
- String inferred_endpoint = DuccUiUtilities.getEndpoint(wd, dd, jvmargs);
+ String inferred_endpoint = DuccUiUtilities.getEndpoint(wd, dd, jvmarg_string);
if (endpoint == null) {
endpoint = inferred_endpoint;
requestProperties.put(UiOption.ServiceRequestEndpoint.pname(), endpoint);
@@ -217,7 +208,7 @@ public class DuccServiceSubmit
return false;
}
- if ( ! resolve_service_dependencies(endpoint) ) {
+ if ( ! check_service_dependencies(endpoint) ) {
return false;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java Fri Aug 2 21:22:58 2013
@@ -25,8 +25,11 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -46,16 +49,22 @@ import org.w3c.dom.NodeList;
public class DuccUiUtilities {
- public static boolean isSupportedBeta() {
- boolean retVal = true;
- String key = DuccPropertiesResolver.ducc_submit_beta;
- String value = DuccPropertiesResolver.get(key);
- if(value != null) {
- if(value.equalsIgnoreCase("off")) {
- retVal = false;
- }
+ private static boolean betaSet = false;
+ private static boolean isBeta;
+
+ public static boolean isSupportedBeta() {
+ if (betaSet) return isBeta;
+
+ // Set DUCC_HOME stsem property (if not already set) for DuccPropertiesResolver
+ Utils.findDuccHome();
+
+ // Only if "on" allow old-style process/driver options
+ isBeta = false;
+ String value = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_submit_beta);
+ if(value != null && value.equalsIgnoreCase("on")) {
+ isBeta = true;
}
- return retVal;
+ return isBeta;
}
public static String getUser() {
@@ -71,37 +80,7 @@ public class DuccUiUtilities {
}
return user;
}
-
- /*
- * Create a map from the user-specified environment string
- * Must be a white-space delimited string of assignments, e.g.
- * TERM=xterm DISPLAY=:1.0 LD_LIBRARY_PATH=/my/own/path EMPTY=
- * Keys & values cannot contain white-space, e.g. all of these will fail:
- * TERM = xterm DISPLAY =:1.0 LD_LIBRARY_PATH="/my/bl nk/path"
- */
- private static Properties environmentMap(String environment) {
- Properties properties = new Properties();
- if (environment.length() == 0) {
- return properties;
- }
- String[] tokens = environment.split("\\s+");
- for (String token : tokens) {
- String[] nvp = token.split("=", 2);
- String key = nvp[0];
- // Reject 'foo' & '=foo' & '=' but accept 'foo='
- if (token.indexOf('=') < 0 || key.length() == 0) {
- return null;
- }
- if (nvp.length > 1) {
- properties.setProperty(key, nvp[1]);
- } else {
- properties.setProperty(key, "");
- }
- }
- // No need to trim properties as the only white-space is between assignments
- return properties;
- }
-
+
public static ArrayList<String> getDuplicateOptions(CommandLine commandLine) {
ArrayList<String> duplicates = new ArrayList<String>();
HashMap<String,String> seen = new HashMap<String,String>();
@@ -136,47 +115,82 @@ public class DuccUiUtilities {
//**********
- public static boolean ducc_environment(CliBase base, Properties jobRequestProperties, String key) {
- boolean retVal = true;
- // Rename the user's LD_LIBRARY_PATH as Secure Linuxs will not pass that on
- String source = "LD_LIBRARY_PATH";
- String target = "DUCC_"+source;
- String environment_string = jobRequestProperties.getProperty(key, "");
- Properties environment_properties = environmentMap(environment_string);
- if (environment_properties == null) {
- base.message("ERROR:", key, "Invalid environment syntax - missing '=' ?");
- return false;
- }
- if(environment_properties.containsKey(source)) {
- if(environment_properties.containsKey(target)) {
- base.message("ERROR:", key, "environment conflict:", target, "takes precedence over", source);
- }
- else {
- environment_properties.setProperty(target, environment_properties.getProperty(source));
- environment_properties.remove(source);
- StringBuilder sb = new StringBuilder();
- for (String name : environment_properties.stringPropertyNames()) {
- sb.append(name).append("=").append(environment_properties.getProperty(name)).append(" ");
- }
- environment_string = sb.toString();
- jobRequestProperties.setProperty(key, environment_string);
- }
- }
- // Augment user-specified environment with a few useful ones, e.g. USER HOME
+ public static String fixupEnvironment(String environment) {
+ // Rename the user's LD_LIBRARY_PATH as Secure Linuxs will not pass that on
+ boolean modified = false;
+ String source = "LD_LIBRARY_PATH";
+ String target = "DUCC_"+source;
+ ArrayList<String> envList = tokenizeList(environment, false); // Don't strip quotes
+ Map<String,String> envMap = parseAssignments(envList, false); // Keep all entries
+ if (envMap == null) {
+ return null; // Syntax error
+ }
+ if (envMap.containsKey(source)) {
+ if (!envMap.containsKey(target)) {
+ envMap.put(target, envMap.get(source));
+ envMap.remove(source);
+ modified = true;
+ }
+ }
+ // Augment user-specified environment with a few useful ones (only if not already set), e.g. USER HOME
+ // If an augmented value contains a blank add single or double quotes
String envNames = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_submit_environment_propagated);
if (envNames != null) {
- StringBuilder sb = new StringBuilder();
for (String name : envNames.split("\\s+")) {
- if (!environment_properties.containsKey(name)) {
- sb.append(name).append("=").append(System.getenv(name)).append(" ");
+ if (!envMap.containsKey(name)) {
+ String value = System.getenv(name);
+ if (value != null) {
+ if (value.indexOf(' ') >= 0) {
+ if (value.indexOf('"') < 0) {
+ value = "\"" + value + "\"";
+ } else if (value.indexOf('\'') < 0) {
+ value = "'" + value + "'";
+ } else {
+ System.out.println("WARNING: omitting environment variable " + name + " as has unquotable value: " + value);
+ continue;
+ }
+ }
+ envMap.put(name, value);
+ modified = true;
+ }
}
}
- if (sb.length() > 0) {
- sb.append(environment_string);
- jobRequestProperties.setProperty(key, sb.toString());
+ }
+ // If changes made rebuild the string ... note that quotes were preserved so can recreate easily
+ if (modified) {
+ StringBuilder sb = new StringBuilder();
+ for (String name : envMap.keySet()) {
+ sb.append(name).append("=").append(envMap.get(name)).append(" ");
}
+ return sb.toString();
+ } else {
+ return environment;
}
- return retVal;
+ }
+
+ public static boolean ducc_environment(CliBase base, Properties jobRequestProperties, String key) {
+ String environment_string = jobRequestProperties.getProperty(key, "");
+ String fixedEnv = fixupEnvironment(environment_string);
+ if (fixedEnv == null) {
+ return false; // error
+ }
+ // If the input string returned unmodified, no need to change the property
+ if (fixedEnv != environment_string) {
+ jobRequestProperties.setProperty(key, fixedEnv);
+ }
+ return true;
+ }
+
+ /*
+ * Get URL for service handling request. Either "orchestrator" or "sm"
+ */
+ public static String dispatchUrl(String server) {
+ String host = DuccPropertiesResolver.get("ducc." + server + ".http.node");
+ String port = DuccPropertiesResolver.get("ducc." + server + ".http.port");
+ if ( host == null || port == null) {
+ throw new IllegalStateException("ducc." + server + ".http.node and/or .port not set in ducc.properties");
+ }
+ return "http://" + host + ":" + port + "/" + server.substring(0, 2);
}
//**********
@@ -290,9 +304,17 @@ public class DuccUiUtilities {
return goodprops;
}
- public static String getEndpoint(String working_dir, String process_DD, Properties jvmargs)
+ /**
+ * Extract the endpoint from the deployment descriptor, resolving names and placeholders against
+ * the same environment as that of the JVM that will deploy the service
+ *
+ * @param working_dir
+ * @param process_DD
+ * @param jvmargs
+ * @return
+ */
+ public static String getEndpoint(String working_dir, String process_DD, String jvmargs)
{
-
// convert relative path for process_DD to absolute if needed
if ( !process_DD.startsWith("/") && process_DD.endsWith(".xml") && working_dir != null ) {
process_DD = working_dir + "/" + process_DD;
@@ -310,16 +332,19 @@ public class DuccUiUtilities {
}
// locate the <inputQueue node within the xml - should only be one such node, and it MUST exist
- // then construct an endpoint and resolve defaultBrokerURL if needed
+ // then construct an endpoint and resolve any placeholders against the process JVM args
+ // just as is done by Spring in a UIMA-AS Deployment Descriptor
NodeList nodes = doc.getElementsByTagName("inputQueue");
if ( nodes.getLength() > 0 ) {
Element element = (Element) nodes.item(0);
String endpoint = element.getAttribute("endpoint");
String broker = element.getAttribute("brokerURL");
String ep = "UIMA-AS:" + endpoint + ":" + broker;
-
- ep = Utils.resolvePlaceholders(ep, jvmargs);
- ep = Utils.resolvePlaceholders(ep, System.getProperties());
+ if (ep.contains("${")) {
+ ArrayList<String> jvmargList = tokenizeList(jvmargs, true); // Strip quotes
+ Map<String, String> jvmargMap = parseAssignments(jvmargList, true); // only -D entries
+ ep = resolvePlaceholders(ep, jvmargMap);
+ }
return ep;
} else {
throw new IllegalArgumentException("Invalid DD:" + process_DD + ". Missing required element <inputQueue ...");
@@ -327,92 +352,189 @@ public class DuccUiUtilities {
}
/**
- * Must resolve ${defaultBrokerURL} from -DdefaultBrokerURL and if it fails, fail the job because
- * the defaut of "tcp://localhost:61616" is never correct for a DUCC job.
+ * Check that dependencies are syntactically correct, and that a service doesn't depend on itself.
*
- * We also split up the string and examine each endpoint so we can fail if invalid endpoints are given.
- *
- * And, we do a quick check for circular dependencies. At this point all we can check for is no duplicates
- * in the set[endpoint, dependencies].
+ * Assumes that any placeholders have been resolved against the caller's environment
*
* @param endpoint This is the endpoint of the caller itself, for resolution ( to make sure it can resolve.). For
* jobs this must be null.
- * @parem dependency_string This is the comma-delimeted string of service ids "I" am dependent upon.
- * @param jvmargs These are the JVM arguments specified for the job, converted to a properties file.
+ * @param dependency_string This is the comma-delimited string of service ids "I" am dependent upon.
*/
- public static String resolve_service_dependencies(String endpoint, String dependency_string, Properties jvmargs)
+ public static void check_service_dependencies(String endpoint, String dependency_string)
{
-
if ( dependency_string == null ) { // no dependencies to worry about
- return null;
+ return;
}
- String[] deplist = dependency_string.split("\\s");
- Map<String, String> resolved = new HashMap<String, String>();
- if ( endpoint != null ) {
- resolved.put(endpoint, endpoint);
- }
- int ndx = 0;
-
- for ( String d : deplist ) {
- d = d.trim();
- if ( d.startsWith(ServiceType.UimaAs.decode() + ":") || d.startsWith(ServiceType.Custom.decode() + ":") ) {
- String nextdep = Utils.resolvePlaceholders(d, jvmargs);
- if ( resolved.containsKey(nextdep) ) {
- throw new IllegalArgumentException("Circular dependencies with " + nextdep);
- }
-
- if ( d.startsWith(ServiceType.UimaAs.decode()) ) {
- // hard to know if this is a good EP or not but se do know that it MUST have 2 ":" in it, and the broker must be a valid url
- // UIMA-AS : queuename : broker
- // This code comes from SM:ServiceSet.parseEndpoint. How best to generalize these?
- ndx = d.indexOf(":");
- if ( ndx <= 0 ) {
- throw new IllegalArgumentException("Invalid UIMA-AS service id: " + d);
- }
+ for (String d : dependency_string.split("\\s+")) {
+ if (d.equals(endpoint)) {
+ throw new IllegalArgumentException("A service cannot depend on itself: " + d);
+ }
+ String[] parts = d.split(":", 3);
+ String type = parts[0];
+ if (!type.equals(ServiceType.UimaAs.decode()) && !type.equals(ServiceType.Custom.decode())) {
+ throw new IllegalArgumentException(
+ "Ill-formed or unsuported service type in dependency: " + d);
+ }
- d = d.substring(ndx+1);
- ndx = d.indexOf(":");
- if ( ndx <= 0 ) {
- throw new IllegalArgumentException("Invalid UIMA-AS service id (missing or invalid broker URL): " + d);
- }
- String qname = d.substring(0, ndx).trim();
- String broker = d.substring(ndx+1).trim();
-
- @SuppressWarnings("unused")
- // this IS unused, it is here only to insure the string is parsed as a URL
- URL url = null;
- try {
- url = new URL(null, broker, new TcpStreamHandler());
- } catch (MalformedURLException e) {
- throw new IllegalArgumentException("Invalid broker URL in service ID: " + broker);
- }
-
- if ( qname.equals("") || broker.equals("") ) {
- throw new IllegalArgumentException("The endpoint cannot be parsed. Expecting UIMA-AS:Endpoint:Broker, received " + d);
- }
-
+ if (type.equals(ServiceType.UimaAs.decode())) {
+ // MUST have 2 ":" in it, and the broker must be a valid url
+ // UIMA-AS:queuename:broker
+ if (parts.length < 3) {
+ throw new IllegalArgumentException("Invalid UIMA-AS service id: " + d);
+ }
+ String qname = parts[1];
+ String broker = parts[2];
+ if (qname.equals("") || broker.equals("")) {
+ throw new IllegalArgumentException("Invalid syntax for UIMA-AS service id: " + d);
+ }
+ // this IS unused, it is here only to insure the string is parsed as a URL
+ @SuppressWarnings("unused")
+ URL url = null;
+ try {
+ url = new URL(null, broker, new TcpStreamHandler());
+ } catch (MalformedURLException e) {
+ throw new IllegalArgumentException("Invalid broker URL in service ID: " + broker);
}
-
- resolved.put(nextdep, nextdep);
- } else {
- throw new IllegalArgumentException("Ill-formed or unsuported service type in dependency: " + d);
}
}
-
- if ( endpoint != null ) {
- resolved.remove(endpoint); // remember to remove "me"!
- }
+ }
+
+ /*
+ * Resolve any ${..} placeholders against a map of JVM arg values
+ */
+ private static String resolvePlaceholders(String contents, Map<String,String> argMap) {
+ // Placeholders syntax ${<placeholder>}
+ Pattern pattern = Pattern.compile("\\$\\{(.*?)\\}"); // Stops on first '}'
+ Matcher matcher = pattern.matcher(contents);
+
StringBuffer sb = new StringBuffer();
- ndx = 0;
- int len = resolved.size();
- for ( String s : resolved.keySet() ) {
- sb.append(s);
- if ( (++ndx ) < len ) {
- sb.append(" ");
+ while (matcher.find()) {
+ final String key = matcher.group(1);
+ String value = argMap.get(key);
+ if (value == null) {
+ throw new IllegalArgumentException("Undefined JVM property '" + key + "' in: " + contents);
}
+ matcher.appendReplacement(sb, value);
}
+ matcher.appendTail(sb);
return sb.toString();
}
+
+ /*
+
+ */
+
+ /**
+ * Create an array of parameters from a whitespace-delimited list (e.g. JVM args or environment assignments.)
+ * Values containing whitespace must be single- or double-quoted:
+ * TERM=xterm DISPLAY=:1.0 LD_LIBRARY_PATH="/my/path/with blanks/" EMPTY= -Dxyz="a b c" -Dabc='x y z'
+ * Quotes may be stripped or preserved.
+ * Values containing both types of quotes are NOT supported.
+ *
+ * @param options
+ * - string of blank-delimited options
+ * @param stripQuotes
+ * - true if balanced quotes are to be removed
+ * @return - array of options
+ */
+ public static ArrayList<String> tokenizeList(String options, boolean stripQuotes) {
+ // Pattern matches a non-quoted region or a double-quoted region or a single-quoted region
+ // 1st part matches one or more non-whitespace characters but not " or '
+ // 2nd part matches a "quoted" region containing any character except "
+ // 3rd part matches a 'quoted' region containing any character except '
+ // See: http://stackoverflow.com/questions/3366281/tokenizing-a-string-but-ignoring-delimiters-within-quotes
+
+ String noSpaceRegex = "[^\\s\"']+";
+ String doubleQuoteRegex = "\"([^\"]*)\"";
+ String singleQuoteRegex = "'([^']*)'";
+ final String regex = noSpaceRegex + "|" + doubleQuoteRegex + "|" + singleQuoteRegex;
+
+ Pattern patn = Pattern.compile(regex);
+ Matcher matcher = patn.matcher(options);
+ ArrayList<String> tokens = new ArrayList<String>();
+ StringBuilder sb = new StringBuilder();
+
+ // If stripping quotes extract the capturing group (without the quotes)
+ // When preserving quotes extract the full region
+ // Combine the pieces of a token until the match ends with whitespace
+ if (stripQuotes) {
+ while (matcher.find()) {
+ if (matcher.group(1) != null) {
+ sb.append(matcher.group(1));
+ } else if (matcher.group(2) != null) {
+ sb.append(matcher.group(2));
+ } else {
+ sb.append(matcher.group());
+ }
+ if (matcher.end() >= options.length() || Character.isWhitespace(options.charAt(matcher.end()))) {
+ tokens.add(sb.toString());
+ sb.setLength(0);
+ }
+ }
+ } else {
+ while (matcher.find()) {
+ sb.append(matcher.group());
+ if (matcher.end() >= options.length() || Character.isWhitespace(options.charAt(matcher.end()))) {
+ tokens.add(sb.toString());
+ sb.setLength(0);
+ }
+ }
+ }
+ return tokens;
+ }
+
+ /*
+ * Create a map from an array of environment variable assignments produced by tokenizeList Quotes
+ * may have been stripped by tokenizeList The value is optional but the key is NOT, e.g. accept
+ * assignments of the form foo=abc & foo= & foo reject =foo & =
+ *
+ * @param assignments - list of environment or JVM arg assignments
+ * @param jvmArgs - true if tokens are JVM args -- process only the -Dprop=value entries
+ *
+ * @return - map of key/value pairs null if syntax is illegal
+ */
+ public static Map<String, String> parseAssignments(List<String> assignments, boolean jvmArgs) {
+
+ HashMap<String, String> map = new HashMap<String, String>();
+ if (assignments == null || assignments.size() == 0) {
+ return map;
+ }
+ for (String assignment : assignments) {
+ String[] parts = assignment.split("=", 2); // Split on first '='
+ String key = parts[0];
+ if (key.length() == 0) {
+ return null;
+ }
+ if (jvmArgs) {
+ if (!key.startsWith("-D")) {
+ continue;
+ }
+ key = key.substring(2);
+ }
+ map.put(key, parts.length > 1 ? parts[1] : "");
+ }
+ return map;
+ }
+ // ====================================================================================================
+
+ /*
+ * Test the quote handling and optional stripping
+ */
+ public static void main(String[] args) {
+ String list = "SINGLE_QUOTED='single quoted'\tDOUBLE_QUOTED=\"double quoted\" SINGLE_QUOTE=\"'\" \r DOUBLE_QUOTE='\"'";
+ System.out.println("List: " + list);
+ System.out.println("\n quotes preserved:");
+ ArrayList<String> tokens = tokenizeList(list, false);
+ for (String token : tokens) {
+ System.out.println("~" + token + "~");
+ }
+
+ System.out.println("\n quotes stripped:");
+ tokens = tokenizeList(list, true);
+ for (String token : tokens) {
+ System.out.println("~" + token + "~");
+ }
+ }
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOne.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOne.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOne.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOne.java Fri Aug 2 21:22:58 2013
@@ -35,9 +35,6 @@ public class AllInOne extends CliBase {
private static String cid = AllInOne.class.getSimpleName();
- private static String or_port = "ducc.orchestrator.http.port";
- private static String or_host = "ducc.orchestrator.http.node";
-
private IMessageHandler mh = new MessageHandler();
private JobRequestProperties jobRequestProperties = new JobRequestProperties();
@@ -49,7 +46,7 @@ public class AllInOne extends CliBase {
public AllInOne(String[] args) throws Exception {
UiOption[] opts = DuccJobSubmit.opts;
- init(this.getClass().getName(), opts, args, jobRequestProperties, or_host, or_port, "or", consoleCb, null);
+ init(this.getClass().getName(), opts, args, jobRequestProperties, consoleCb);
}
private void examine_debug() {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java Fri Aug 2 21:22:58 2013
@@ -20,7 +20,6 @@ package org.apache.uima.ducc.cli.aio;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -44,9 +43,6 @@ public class AllInOneLauncher extends Cl
private static String cid = AllInOneLauncher.class.getSimpleName();
- private static String or_port = "ducc.orchestrator.http.port";
- private static String or_host = "ducc.orchestrator.http.node";
-
private static String remote = "remote";
private static String local = "local";
@@ -107,13 +103,13 @@ public class AllInOneLauncher extends Cl
public AllInOneLauncher(String[] args) throws Exception {
this.args = args;
- init(this.getClass().getName(), opts, args, jobRequestProperties, or_host, or_port, "or", null, null);
+ init (this.getClass().getName(), opts, args, jobRequestProperties, null);
}
public AllInOneLauncher(String[] args, IDuccCallback consoleCb) throws Exception {
this.args = args;
mh = new MessageHandler(consoleCb);
- init(this.getClass().getName(), opts, args, jobRequestProperties, or_host, or_port, "or", consoleCb, null);
+ init (this.getClass().getName(), opts, args, jobRequestProperties, consoleCb);
}
private boolean isLocal() {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java?rev=1509867&r1=1509866&r2=1509867&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java Fri Aug 2 21:22:58 2013
@@ -39,6 +39,11 @@ public class DuccPropertiesResolver {
return duccPropertiesResolver.getProperty(key);
}
+ public static String get(String key, String dflt) {
+ String value = duccPropertiesResolver.getProperty(key);
+ return value != null ? value : dflt;
+ }
+
private Properties initialProperties = new DuccProperties();
private Properties defaultProperties = new DuccProperties();
@@ -68,7 +73,7 @@ public class DuccPropertiesResolver {
public static final String ducc_orchestrator_state_update_endpoint_type = "ducc.orchestrator.state.update.endpoint.type";
public static final String ducc_orchestrator_state_update_endpoint = "ducc.orchestrator.state.update.endpoint";
public static final String ducc_orchestrator_http_port = "ducc.orchestrator.http.port";
- public static final String ducc_orchestrator_node = "ducc.orchestrator.http.node";
+ public static final String ducc_orchestrator_http_node = "ducc.orchestrator.http.node";
public static final String ducc_orchestrator_maintenance_rate = "ducc.orchestrator.maintenance.rate";
public static final String ducc_orchestrator_job_factory_classpath_order = "ducc.orchestrator.job.factory.classpath.order";