You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by bu...@apache.org on 2013/10/03 21:03:54 UTC
svn commit: r1528974 - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/
uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/
uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/
Author: burn
Date: Thu Oct 3 19:03:54 2013
New Revision: 1528974
URL: http://svn.apache.org/r1528974
Log:
UIMA-3312 Added support for --suppress_console_logging; --attach_console replaces the separate driver/process_attach_console options
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-process-submit.tex
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java?rev=1528974&r1=1528973&r2=1528974&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/CliBase.java Thu Oct 3 19:03:54 2013
@@ -333,12 +333,9 @@ public abstract class CliBase
setWorkingDirectory();
setUser();
- //TODO - shouldn't environment fixups be done here for all requests that may use it??
-
NodeIdentity ni = new NodeIdentity();
- host_address = ni.getIp();
-
-
+ host_address = ni.getIp();
+
initConsoleListener();
String targetUrl = DuccUiUtilities.dispatchUrl(servlet);
@@ -371,6 +368,7 @@ public abstract class CliBase
/*
* Check for missing required options, set defaults, and validate where possible
+ * Also fixup the environment for all that use it.
*/
void setDefaults(UiOption[] uiOpts, boolean suppress_console) throws Exception {
for (UiOption uiopt : uiOpts) {
@@ -392,10 +390,9 @@ public abstract class CliBase
}
}
// If this request accepts the --environment option may need to augment it by
- // renaming LD_LIBRARY_PATH, propagating some user values, disabling the console log
+ // renaming LD_LIBRARY_PATH & propagating some user values
if (uiopt == UiOption.Environment) {
- String addEnv = suppress_console ? " DUCC_SUPPRESS_CONSOLE=true" : "";
- DuccUiUtilities.ducc_environment(this, cli_props, addEnv);
+ DuccUiUtilities.ducc_environment(this, cli_props);
}
}
}
@@ -582,32 +579,6 @@ public abstract class CliBase
return debug;
}*/
- /**
- * Set the internal API debug flag.
- * @param val Set to true to enable debugging, and false to disable it.
- */
-/* public void setDebug(boolean val)
- {
- this.debug = val;
- }
-*/
- // nobody seems to use this
-// public String getHostAddress()
-// {
-// return host_address;
-// }
-
-/* Also unused?
- * public boolean hasProperty(String key)
- {
- return cli_props.containsKey(key);
- }
-
- public String getProperty(String key)
- {
- return (String) cli_props.getProperty(key);
- }*/
-
protected IDuccCallback getCallback()
{
return consoleCb;
@@ -710,29 +681,30 @@ public abstract class CliBase
/**
* Needs to be done before submitting the job because the job needs the ports. We'll
- * just define the listener, but not start it untile the job monitor starts, in case the
+ * just define the listener, but not start it until the job monitor starts, in case the
* submission fails.
*/
- protected void initConsoleListener()
- throws Exception
- {
- console_attach =
- cli_props.containsKey(UiOption.ProcessAttachConsole.pname()) ||
- cli_props.containsKey(UiOption.DriverAttachConsole.pname());
+ protected void initConsoleListener() throws Exception {
+ String value;
- if ( console_attach ) {
+ console_attach = cli_props.containsKey(UiOption.AttachConsole.pname());
+ if (console_attach) {
console_listener = new ConsoleListener(this, consoleCb);
-
- String key = UiOption.Environment.pname();
-
- if ( cli_props.containsKey(UiOption.ProcessAttachConsole.pname()) ) {
- set_console_port(cli_props, key);
- }
-
- if (cli_props.containsKey(UiOption.DriverAttachConsole.pname()) ) {
- set_console_port(cli_props, key);
- }
+ value = console_listener.getConsoleHostAddress();
+ } else if (suppress_console_log) {
+ value = "suppress";
+ } else {
+ return;
+ }
+ // Set the console "suppress" flag or the host:port for the console listener into the env
+ String key = UiOption.Environment.pname();
+ String env = cli_props.getProperty(key);
+ if (env == null) {
+ env = "DUCC_CONSOLE_LISTENER=" + value;
+ } else {
+ env += " DUCC_CONSOLE_LISTENER=" + value;
}
+ cli_props.setProperty(key, env);
}
/**
@@ -763,31 +735,6 @@ public abstract class CliBase
}
}
- protected void set_console_port(DuccProperties props, String key)
- {
- if ( key != null ) {
- if ( console_listener == null ) {
- message("WARN: Attempt to set console port but listener is not running.");
- return;
- }
-
- String console_host_address = console_listener.getConsoleHostAddress();
- int console_listener_port = console_listener.getConsolePort();
-
- String envval = "DUCC_CONSOLE_LISTENER";
- String env = props.getProperty(key);
- // Set the host:port for the console listener into the env
- String console_address = console_host_address + ":" + console_listener_port;
- String dp = envval + "=" + console_address;
- if ( env == null ) {
- env = dp;
- } else {
- env = env + " " + dp;
- }
- props.setProperty(key, env);
- }
- }
-
/**
* This is used to find if the remote console is redirected to the local process, and if so, is it still
* active.
@@ -868,6 +815,8 @@ public abstract class CliBase
args[i] = "--classpath";
} else if (arg.equals("--driver_environment") || arg.equals("--process_environment")) {
args[i] = "--environment";
+ } else if (arg.equals("--driver_attach_console") || arg.equals("--process_attach_console")) {
+ args[i] = "--attach_console";
} else if (arg.equals("--cancel_job_on_interrupt") || arg.equals("--cancel_managed_reservation_on_interrupt")) {
args[i] = "--cancel_on_interrupt";
} else if (arg.equals("--jvm_args")) {
@@ -892,6 +841,9 @@ public abstract class CliBase
} else if (key.equals("driver_environment") || key.equals("process_environment")) {
props.put("environment", props.get(key));
props.remove(key);
+ } else if (key.equals("driver_attach_console") || key.equals("process_attach_console")) {
+ props.put("attach_console", props.get(key));
+ props.remove(key);
} else if (key.equals("cancel_job_on_interrupt") || key.equals("cancel_managed_reservation_on_interrupt")) {
props.put("cancel_on_interrupt", props.get(key));
props.remove(key);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java?rev=1528974&r1=1528973&r2=1528974&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ConsoleListener.java Thu Oct 3 19:03:54 2013
@@ -40,7 +40,6 @@ class ConsoleListener
private CliBase submit;
private Map<Integer, Pair<StdioReader, StdioWriter>> listeners = new HashMap<Integer, Pair<StdioReader, StdioWriter>>();
- private int console_listener_port;
private String console_host_address;
private boolean in_shutdown = false;
@@ -58,11 +57,13 @@ class ConsoleListener
{
this.submit = submit;
this.sock = new ServerSocket(0);
- this.console_listener_port = sock.getLocalPort();
this.consoleCb = consoleCb;
NodeIdentity ni = new NodeIdentity();
- this.console_host_address = ni.getIp();
+ String host_address = ni.getIp();
+ int console_listener_port = sock.getLocalPort();
+ this.console_host_address = host_address + ":" + console_listener_port;
+
debug = submit.debug; //isDebug();
suppress_log = submit.suppress_console_log;
@@ -73,11 +74,6 @@ class ConsoleListener
return console_host_address;
}
- int getConsolePort()
- {
- return console_listener_port;
- }
-
synchronized boolean isShutdown()
{
return in_shutdown;
@@ -128,7 +124,7 @@ class ConsoleListener
public void run()
{
- if ( debug ) System.out.println("Listening on " + console_host_address + " " + console_listener_port);
+ if ( debug ) System.out.println("Listening on " + console_host_address);
while ( true ) {
try {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java?rev=1528974&r1=1528973&r2=1528974&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java Thu Oct 3 19:03:54 2013
@@ -22,7 +22,6 @@ import java.lang.management.ManagementFa
import java.util.ArrayList;
import java.util.Properties;
-import org.apache.commons.cli.MissingArgumentException;
import org.apache.uima.ducc.cli.aio.AllInOneLauncher;
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
import org.apache.uima.ducc.common.utils.DuccSchedulerClasses;
@@ -46,11 +45,10 @@ public class DuccJobSubmit
UiOption.Timestamp,
UiOption.AllInOne,
-
+
+ UiOption.AttachConsole,
UiOption.ProcessDebug,
- UiOption.ProcessAttachConsole,
UiOption.DriverDebug,
- UiOption.DriverAttachConsole,
UiOption.Description,
UiOption.SchedulingClass,
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java?rev=1528974&r1=1528973&r2=1528974&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java Thu Oct 3 19:03:54 2013
@@ -21,7 +21,6 @@ package org.apache.uima.ducc.cli;
import java.util.ArrayList;
import java.util.Properties;
-import org.apache.uima.ducc.cli.IUiOptions.UiOption;
import org.apache.uima.ducc.transport.event.IDuccContext.DuccContext;
import org.apache.uima.ducc.transport.event.SubmitServiceDuccEvent;
import org.apache.uima.ducc.transport.event.SubmitServiceReplyDuccEvent;
@@ -46,7 +45,7 @@ public class DuccManagedReservationSubmi
UiOption.Description,
UiOption.Environment,
UiOption.LogDirectory,
- UiOption.ProcessAttachConsole,
+ UiOption.AttachConsole,
UiOption.ProcessExecutable,
UiOption.ProcessExecutableArgs,
UiOption.ProcessFailuresLimit,
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java?rev=1528974&r1=1528973&r2=1528974&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccUiUtilities.java Thu Oct 3 19:03:54 2013
@@ -112,12 +112,12 @@ public class DuccUiUtilities {
}
}
- public static void ducc_environment(CliBase base, Properties jobRequestProperties, String extraEnv) {
+ public static void ducc_environment(CliBase base, Properties jobRequestProperties) {
String key = UiOption.Environment.pname();
String environment_string = jobRequestProperties.getProperty(key, "");
- String fixedEnv = fixupEnvironment(environment_string + extraEnv);
- // If the input string returned unmodified, no need to change the property
- if (!fixedEnv.equals(environment_string)) {
+ String fixedEnv = fixupEnvironment(environment_string);
+ // If the same string returned, no need to change the property as it was not modified
+ if (fixedEnv != environment_string) {
jobRequestProperties.setProperty(key, fixedEnv);
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java?rev=1528974&r1=1528973&r2=1528974&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java Thu Oct 3 19:03:54 2013
@@ -63,6 +63,14 @@ public interface IUiOptions
public String label() { return name(); }
},
+ AttachConsole {
+ public String pname() { return "attach_console"; }
+ public String argname() { return null; }
+ public String description() { return "If specified, redirect remote stdout and stderr to the local submitting console."; }
+ public String example() { return null; }
+ public String label() { return name(); }
+ },
+
Autostart {
public String pname() { return "autostart"; }
public String argname() { return "boolean: true or false"; }
@@ -130,14 +138,6 @@ public interface IUiOptions
public String label() { return "Description"; }
},
- DriverAttachConsole {
- public String pname() { return "driver_attach_console"; }
- public String argname() { return null; }
- public String description() { return "If specified, redirect remote job driver stdout and stderr to the local submitting console."; }
- public String example() { return null; }
- public String label() { return "DriverAttachConsole"; }
- },
-
DriverDescriptorCR {
public String pname() { return JobSpecificationProperties.key_driver_descriptor_CR; }
public String description() { return "Driver (collection reader) descriptor."; }
@@ -370,14 +370,6 @@ public interface IUiOptions
public String label() { return null; }
},
- ProcessAttachConsole {
- public String pname() { return "process_attach_console"; }
- public String argname() { return null; }
- public String description() { return "If specified, redirect remote process stdout and stderr to the local submitting console."; }
- public String example() { return null; }
- public String label() { return "ProcessAttachConsole"; }
- },
-
ProcessDebug {
public String pname() { return "process_debug"; }
public String argname() { return "debugger-port-number"; }
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java?rev=1528974&r1=1528973&r2=1528974&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java Thu Oct 3 19:03:54 2013
@@ -232,7 +232,7 @@ public class AllInOneLauncher extends Cl
private void examine_process_attach_console() {
String mid = "examine_process_attach_console";
mh.frameworkTrace(cid, mid, enter);
- String pname = UiOption.ProcessAttachConsole.pname();
+ String pname = UiOption.AttachConsole.pname();
if(jobRequestProperties.containsKey(pname)) {
String message = "attach_console";
mh.frameworkDebug(cid, mid, message);
@@ -858,7 +858,7 @@ public class AllInOneLauncher extends Cl
if(cancel_on_interrupt) {
addArg(cmdLine, "--"+UiOption.CancelOnInterrupt.pname());
}
- addArg(cmdLine, "--"+UiOption.ProcessAttachConsole.pname()); // Always return console output to match "local"
+ addArg(cmdLine, "--"+UiOption.AttachConsole.pname()); // Always return console output to match "local"
String[] argList = cmdLine.toArray(new String[cmdLine.size()]);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-process-submit.tex
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-process-submit.tex?rev=1528974&r1=1528973&r2=1528974&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-process-submit.tex (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-process-submit.tex Thu Oct 3 19:03:54 2013
@@ -13,7 +13,7 @@
the necessary resources (and prevent over-allocation), and the DUCC run-time environment
to manage process lifetime.
- If {\em process\_attach\_console} is specified, Stdin, Stderr, and Stdout of the remote
+ If {\em attach\_console} is specified, Stdin, Stderr, and Stdout of the remote
process are redirected to the submitting console. It is thus possible to run interactive
sessions with remote processes where the resources are managed by DUCC.
@@ -26,6 +26,9 @@
\paragraph{Options:}
\begin{description}
+ \item[$--$attach\_console] If specified, remote process stdout and stderr are
+ redirected to, and stdin redirected from, the local submitting console.
+
\item[$--$cancel\_on\_interrupt ] Cancel managed reservation on interrupt
(Ctrl-C). If running with {\em $--$wait\_for\_completion} and this flag is specified,
terminating the submit process will result in the remote process being terminated.
@@ -68,9 +71,6 @@
machines DUCC, sub-directory and file names are generated by DUCC and may
not be directly specified.
- \item[$--$process\_attach\_console] If specified, redirect remote process stdio is
- redirected the local submitting console.
-
\item[$--$process\_executable {[program name]}] This is the full path to a program to be
executed.
@@ -108,6 +108,9 @@ scheduling_class = fixed
process_memory_size = 15
\end{verbatim}
+ \item[$--$suppress\_console\_log] If specified, suppress creation of the log files that
+ normally hold the redirected stdout and stderr.
+
\item[$--$wait\_for\_completion ] If specified, the submit command does not return control to
the console immediately, and instead monitors the DUCC state traffic and prints
information about the process as it progresses.
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex?rev=1528974&r1=1528973&r2=1528974&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex Thu Oct 3 19:03:54 2013
@@ -27,6 +27,9 @@
If {\em remote} is specified, the jobs is submitted to DUCC as a {\em manged reservation}
and run on some (presumably larger) machine allocated by DUCC.
+ \item[$--$attach\_console] If specified, redirect remote stdout and stderr
+ to the local submitting console.
+
\item[$--$cancel\_on\_interrupt]. If the job is started with $--$wait\_for\_completion, this
option causes the job to be canceled if the submit command is terminated,
e.g., with CTL-C. If $--$cancel\_job\_on\_interrupt is not
@@ -48,7 +51,7 @@
displayed in the Web Server. When specified on a command-line the text usually
must be surrounded by quotes to protect it from the shell. The default is ``none''.
- \item[$--$driver\_attach\_console] If specified, redirect remote job driver stdout and stderr
+ \item[$--$attach\_console] If specified, redirect remote stdout and stderr
to the local submitting console.
\item[$--$driver\_debug {[debugger-address]}] Append JVM debug flags to the JVM arguments
@@ -146,9 +149,6 @@
machines, sub-directory and file names are generated by DUCC and may
not be directly specified.
- \item[$--$process\_attach\_console] If specified, redirect remote process (as
- opposed to driver) stdout and stderr to the local submitting console.
-
\item[$--$process\_DD {[DD descriptor]} ]
This specifies a UIMA Deployment Descriptor for the job processes for DD-style jobs.
@@ -333,6 +333,9 @@ process_memory_size = 15
Note that properties in a specifications file may be overridden by other command-line
parameters, as discussed \hyperref[chap:cli]{here}.
+ \item[$--$suppress\_console\_log] If specified, suppress creation of the log files that
+ normally hold the redirected stdout and stderr.
+
\item[$--$time-stamp ]
If specified, messages from the submit process are timestamped. This is intended primarily