You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/07/16 18:59:58 UTC
systemml git commit: [MINOR] Refactoring and cleanup CLI arguments
parsing
Repository: systemml
Updated Branches:
refs/heads/master cffefca30 -> 5aadb4b22
[MINOR] Refactoring and cleanup CLI arguments parsing
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/5aadb4b2
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/5aadb4b2
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/5aadb4b2
Branch: refs/heads/master
Commit: 5aadb4b2204b9238e73025c837a1aa3ed2908fe4
Parents: cffefca
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Jul 16 11:48:53 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Jul 16 11:48:53 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/api/DMLOptions.java | 273 +++++++
.../java/org/apache/sysml/api/DMLScript.java | 268 +------
.../sysml/api/mlcontext/ScriptExecutor.java | 2 +-
.../sysml/test/unit/CLIOptionsParserTest.java | 728 +++++++++----------
4 files changed, 627 insertions(+), 644 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/5aadb4b2/src/main/java/org/apache/sysml/api/DMLOptions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLOptions.java b/src/main/java/org/apache/sysml/api/DMLOptions.java
new file mode 100644
index 0000000..1f0f45a
--- /dev/null
+++ b/src/main/java/org/apache/sysml/api/DMLOptions.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.api;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.PosixParser;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.api.mlcontext.ScriptType;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.utils.Explain;
+import org.apache.sysml.utils.Explain.ExplainType;
+
+/**
+ * Set of DMLOptions that can be set through the command line
+ * and {@link org.apache.sysml.api.mlcontext.MLContext}
+ * The values have been initialized with the default values
+ * Despite there being a DML and PyDML, this class is named DMLOptions
+ * to keep it consistent with {@link DMLOptions} and {@link DMLOptions}
+ */
+public class DMLOptions {
+ public final Options options;
+ public Map<String, String> argVals = new HashMap<>(); // Arguments map containing either named arguments or arguments by position for a DML program
+ public String configFile = null; // Path to config file if default config and default config is to be overriden
+ public boolean clean = false; // Whether to clean up all SystemML working directories (FS, DFS)
+ public boolean stats = false; // Whether to record and print the statistics
+ public int statsCount = 10; // Default statistics count
+ public boolean memStats = false; // max memory statistics
+ public Explain.ExplainType explainType = Explain.ExplainType.NONE; // Whether to print the "Explain" and if so, what type
+ public RUNTIME_PLATFORM execMode = OptimizerUtils.getDefaultExecutionMode(); // Execution mode standalone, MR, Spark or a hybrid
+ public boolean gpu = false; // Whether to use the GPU
+ public boolean forceGPU = false; // Whether to ignore memory & estimates and always use the GPU
+ public boolean debug = false; // to go into debug mode to be able to step through a program
+ public ScriptType scriptType = ScriptType.DML; // whether the script is a DML or PyDML script
+ public String filePath = null; // path to script
+ public String script = null; // the script itself
+ public boolean help = false; // whether to print the usage option
+
+ public final static DMLOptions defaultOptions = new DMLOptions(null);
+
+ public DMLOptions(Options opts) {
+ options = opts;
+ }
+
+ @Override
+ public String toString() {
+ return "DMLOptions{" +
+ "argVals=" + argVals +
+ ", configFile='" + configFile + '\'' +
+ ", clean=" + clean +
+ ", stats=" + stats +
+ ", statsCount=" + statsCount +
+ ", memStats=" + memStats +
+ ", explainType=" + explainType +
+ ", execMode=" + execMode +
+ ", gpu=" + gpu +
+ ", forceGPU=" + forceGPU +
+ ", debug=" + debug +
+ ", scriptType=" + scriptType +
+ ", filePath='" + filePath + '\'' +
+ ", script='" + script + '\'' +
+ ", help=" + help +
+ '}';
+ }
+
+
+ /**
+ * Parses command line arguments to create a {@link DMLOptions} instance with the correct options
+ * @param args arguments from the command line
+ * @param options an {@link Options} instance containing the options that need to be parsed
+ * @return an instance of {@link Options} that contain the correct {@link Option}s.
+ * @throws org.apache.commons.cli.ParseException if there is an incorrect option specified in the CLI
+ */
+ public static DMLOptions parseCLArguments(String[] args)
+ throws org.apache.commons.cli.ParseException
+ {
+ Options options = createCLIOptions();
+ CommandLineParser clParser = new PosixParser();
+ CommandLine line = clParser.parse(options, args);
+
+ DMLOptions dmlOptions = new DMLOptions(options);
+ dmlOptions.help = line.hasOption("help");
+ dmlOptions.scriptType = line.hasOption("python") ? ScriptType.PYDML : ScriptType.DML;
+ dmlOptions.debug = line.hasOption("debug");
+ dmlOptions.gpu = line.hasOption("gpu");
+ if (dmlOptions.gpu) {
+ String force = line.getOptionValue("gpu");
+ if (force != null) {
+ if (force.equalsIgnoreCase("force")) {
+ dmlOptions.forceGPU = true;
+ } else {
+ throw new org.apache.commons.cli.ParseException("Invalid argument specified for -gpu option");
+ }
+ }
+ }
+ if (line.hasOption("exec")){
+ String execMode = line.getOptionValue("exec");
+ if (execMode != null){
+ if (execMode.equalsIgnoreCase("hadoop")) dmlOptions.execMode = RUNTIME_PLATFORM.HADOOP;
+ else if (execMode.equalsIgnoreCase("singlenode")) dmlOptions.execMode = RUNTIME_PLATFORM.SINGLE_NODE;
+ else if (execMode.equalsIgnoreCase("hybrid")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID;
+ else if (execMode.equalsIgnoreCase("hybrid_spark")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID_SPARK;
+ else if (execMode.equalsIgnoreCase("spark")) dmlOptions.execMode = RUNTIME_PLATFORM.SPARK;
+ else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -exec option, must be one of [hadoop, singlenode, hybrid, hybrid_spark, spark]");
+ }
+ }
+ if (line.hasOption("explain")) {
+ dmlOptions.explainType = ExplainType.RUNTIME;
+ String explainType = line.getOptionValue("explain");
+ if (explainType != null){
+ if (explainType.equalsIgnoreCase("hops")) dmlOptions.explainType = ExplainType.HOPS;
+ else if (explainType.equalsIgnoreCase("runtime")) dmlOptions.explainType = ExplainType.RUNTIME;
+ else if (explainType.equalsIgnoreCase("recompile_hops")) dmlOptions.explainType = ExplainType.RECOMPILE_HOPS;
+ else if (explainType.equalsIgnoreCase("recompile_runtime")) dmlOptions.explainType = ExplainType.RECOMPILE_RUNTIME;
+ else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime]");
+ }
+ }
+ dmlOptions.stats = line.hasOption("stats");
+ if (dmlOptions.stats){
+ String statsCount = line.getOptionValue("stats");
+ if (statsCount != null) {
+ try {
+ dmlOptions.statsCount = Integer.parseInt(statsCount);
+ } catch (NumberFormatException e) {
+ throw new org.apache.commons.cli.ParseException("Invalid argument specified for -stats option, must be a valid integer");
+ }
+ }
+ }
+ dmlOptions.memStats = line.hasOption("mem");
+
+ dmlOptions.clean = line.hasOption("clean");
+
+ if (line.hasOption("config")){
+ dmlOptions.configFile = line.getOptionValue("config");
+ }
+
+ if (line.hasOption("f")){
+ dmlOptions.filePath = line.getOptionValue("f");
+ }
+
+ if (line.hasOption("s")){
+ dmlOptions.script = line.getOptionValue("s");
+ }
+
+ // Positional arguments map is created as ("$1", "a"), ("$2", 123), etc
+ if (line.hasOption("args")){
+ String[] argValues = line.getOptionValues("args");
+ for (int k=0; k<argValues.length; k++){
+ String str = argValues[k];
+ if (!str.isEmpty()) {
+ dmlOptions.argVals.put("$" + (k+1), str);
+ }
+ }
+ }
+
+ // Named arguments map is created as ("$K, 123), ("$X", "X.csv"), etc
+ if (line.hasOption("nvargs")){
+ String varNameRegex = "^[a-zA-Z]([a-zA-Z0-9_])*$";
+ String[] nvargValues = line.getOptionValues("nvargs");
+ for (String str : nvargValues){
+ if (!str.isEmpty()){
+ String[] kv = str.split("=");
+ if (kv.length != 2){
+ throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, must be a list of space separated K=V pairs, where K is a valid name of a variable in the DML/PyDML program");
+ }
+ if (!kv[0].matches(varNameRegex)) {
+ throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, " + kv[0] + " does not seem like a valid variable name in DML. Valid variable names in DML start with upper-case or lower-case letter, and contain only letters, digits, or underscores");
+ }
+ dmlOptions.argVals.put("$" + kv[0], kv[1]);
+ }
+ }
+ }
+
+ return dmlOptions;
+ }
+
+ /**
+ * Creates an {@link Options} instance for the command line parameters
+ * As of SystemML 0.13, Apache Commons CLI 1.2 is transitively in the classpath
+ * However the most recent version of Apache Commons CLI is 1.4
+ * Creating CLI options is done using Static methods. Instead of {@link OptionBuilder},
+ * CLI 1.4 uses Option.Builder which has non-static methods.
+ * @return an appropriate instance of {@link Options}
+ */
+ @SuppressWarnings("static-access")
+ private static Options createCLIOptions() {
+ Options options = new Options();
+ Option nvargsOpt = OptionBuilder.withArgName("key=value")
+ .withDescription("parameterizes DML script with named parameters of the form <key=value>; <key> should be a valid identifier in DML/PyDML")
+ .hasArgs().create("nvargs");
+ Option argsOpt = OptionBuilder.withArgName("argN")
+ .withDescription("specifies positional parameters; first value will replace $1 in DML program; $2 will replace 2nd and so on")
+ .hasArgs().create("args");
+ Option configOpt = OptionBuilder.withArgName("filename")
+ .withDescription("uses a given configuration file (can be on local/hdfs/gpfs; default values in SystemML-config.xml")
+ .hasArg().create("config");
+ Option cleanOpt = OptionBuilder.withDescription("cleans up all SystemML working directories (FS, DFS); all other flags are ignored in this mode. \n")
+ .create("clean");
+ Option statsOpt = OptionBuilder.withArgName("count")
+ .withDescription("monitors and reports summary execution statistics; heavy hitter <count> is 10 unless overridden; default off")
+ .hasOptionalArg().create("stats");
+ Option memOpt = OptionBuilder.withDescription("monitors and reports max memory consumption in CP; default off")
+ .create("mem");
+ Option explainOpt = OptionBuilder.withArgName("level")
+ .withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime'")
+ .hasOptionalArg().create("explain");
+ Option execOpt = OptionBuilder.withArgName("mode")
+ .withDescription("sets execution mode; can be 'hadoop' / 'singlenode' / 'hybrid'[default] / 'hybrid_spark' / 'spark'")
+ .hasArg().create("exec");
+ Option gpuOpt = OptionBuilder.withArgName("force")
+ .withDescription("uses CUDA instructions when reasonable; set <force> option to skip conservative memory estimates and use GPU wherever possible; default off")
+ .hasOptionalArg().create("gpu");
+ Option debugOpt = OptionBuilder.withDescription("runs in debug mode; default off")
+ .create("debug");
+ Option pythonOpt = OptionBuilder.withDescription("parses Python-like DML")
+ .create("python");
+ Option fileOpt = OptionBuilder.withArgName("filename")
+ .withDescription("specifies dml/pydml file to execute; path can be local/hdfs/gpfs (prefixed with appropriate URI)")
+ .isRequired().hasArg().create("f");
+ Option scriptOpt = OptionBuilder.withArgName("script_contents")
+ .withDescription("specified script string to execute directly")
+ .isRequired().hasArg().create("s");
+ Option helpOpt = OptionBuilder.withDescription("shows usage message")
+ .create("help");
+
+ options.addOption(configOpt);
+ options.addOption(cleanOpt);
+ options.addOption(statsOpt);
+ options.addOption(memOpt);
+ options.addOption(explainOpt);
+ options.addOption(execOpt);
+ options.addOption(gpuOpt);
+ options.addOption(debugOpt);
+ options.addOption(pythonOpt);
+
+ // Either a clean(-clean), a file(-f), a script(-s) or help(-help) needs to be specified
+ OptionGroup fileOrScriptOpt = new OptionGroup()
+ .addOption(scriptOpt).addOption(fileOpt).addOption(cleanOpt).addOption(helpOpt);
+ fileOrScriptOpt.setRequired(true);
+ options.addOptionGroup(fileOrScriptOpt);
+
+ // Either -args or -nvargs
+ options.addOptionGroup(new OptionGroup()
+ .addOption(nvargsOpt).addOption(argsOpt));
+ options.addOption(helpOpt);
+
+ return options;
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/5aadb4b2/src/main/java/org/apache/sysml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java
index 9c8a3eb..50a23aa 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -31,20 +31,12 @@ import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Scanner;
import org.apache.commons.cli.AlreadySelectedException;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -117,55 +109,7 @@ public class DMLScript
// ARC, // https://dbs.uni-leipzig.de/file/ARC.pdf
// LOOP_AWARE // different policies for operations in for/while/parfor loop vs out-side the loop
}
-
- /**
- * Set of DMLOptions that can be set through the command line
- * and {@link org.apache.sysml.api.mlcontext.MLContext}
- * The values have been initialized with the default values
- * Despite there being a DML and PyDML, this class is named DMLOptions
- * to keep it consistent with {@link DMLScript} and {@link DMLOptions}
- */
- public static class DMLOptions {
- public Map<String, String> argVals = new HashMap<>(); // Arguments map containing either named arguments or arguments by position for a DML program
- public String configFile = null; // Path to config file if default config and default config is to be overriden
- public boolean clean = false; // Whether to clean up all SystemML working directories (FS, DFS)
- public boolean stats = false; // Whether to record and print the statistics
- public int statsCount = 10; // Default statistics count
- public boolean memStats = false; // max memory statistics
- public Explain.ExplainType explainType = Explain.ExplainType.NONE; // Whether to print the "Explain" and if so, what type
- public DMLScript.RUNTIME_PLATFORM execMode = OptimizerUtils.getDefaultExecutionMode(); // Execution mode standalone, MR, Spark or a hybrid
- public boolean gpu = false; // Whether to use the GPU
- public boolean forceGPU = false; // Whether to ignore memory & estimates and always use the GPU
- public boolean debug = false; // to go into debug mode to be able to step through a program
- public ScriptType scriptType = ScriptType.DML; // whether the script is a DML or PyDML script
- public String filePath = null; // path to script
- public String script = null; // the script itself
- public boolean help = false; // whether to print the usage option
-
- public final static DMLOptions defaultOptions = new DMLOptions();
-
- @Override
- public String toString() {
- return "DMLOptions{" +
- "argVals=" + argVals +
- ", configFile='" + configFile + '\'' +
- ", clean=" + clean +
- ", stats=" + stats +
- ", statsCount=" + statsCount +
- ", memStats=" + memStats +
- ", explainType=" + explainType +
- ", execMode=" + execMode +
- ", gpu=" + gpu +
- ", forceGPU=" + forceGPU +
- ", debug=" + debug +
- ", scriptType=" + scriptType +
- ", filePath='" + filePath + '\'' +
- ", script='" + script + '\'' +
- ", help=" + help +
- '}';
- }
- }
-
+
public static RUNTIME_PLATFORM rtplatform = DMLOptions.defaultOptions.execMode; // the execution mode
public static boolean STATISTICS = DMLOptions.defaultOptions.stats; // whether to print statistics
public static boolean FINEGRAINED_STATISTICS = false; // whether to print fine-grained statistics
@@ -258,202 +202,6 @@ public class DMLScript
}
/**
- * Parses command line arguments to create a {@link DMLOptions} instance with the correct options
- * @param args arguments from the command line
- * @param options an {@link Options} instance containing the options that need to be parsed
- * @return an instance of {@link Options} that contain the correct {@link Option}s.
- * @throws org.apache.commons.cli.ParseException if there is an incorrect option specified in the CLI
- */
- public static DMLOptions parseCLArguments(String[] args, Options options) throws org.apache.commons.cli.ParseException {
-
- CommandLineParser clParser = new PosixParser();
- CommandLine line = clParser.parse(options, args);
-
- DMLOptions dmlOptions = new DMLOptions();
- dmlOptions.help = line.hasOption("help");
- dmlOptions.scriptType = line.hasOption("python") ? ScriptType.PYDML : ScriptType.DML;
- dmlOptions.debug = line.hasOption("debug");
- dmlOptions.gpu = line.hasOption("gpu");
- if (dmlOptions.gpu) {
- String force = line.getOptionValue("gpu");
- if (force != null) {
- if (force.equalsIgnoreCase("force")) {
- dmlOptions.forceGPU = true;
- } else {
- throw new org.apache.commons.cli.ParseException("Invalid argument specified for -gpu option");
- }
- }
- }
- if (line.hasOption("exec")){
- String execMode = line.getOptionValue("exec");
- if (execMode != null){
- if (execMode.equalsIgnoreCase("hadoop")) dmlOptions.execMode = RUNTIME_PLATFORM.HADOOP;
- else if (execMode.equalsIgnoreCase("singlenode")) dmlOptions.execMode = RUNTIME_PLATFORM.SINGLE_NODE;
- else if (execMode.equalsIgnoreCase("hybrid")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID;
- else if (execMode.equalsIgnoreCase("hybrid_spark")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID_SPARK;
- else if (execMode.equalsIgnoreCase("spark")) dmlOptions.execMode = RUNTIME_PLATFORM.SPARK;
- else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -exec option, must be one of [hadoop, singlenode, hybrid, hybrid_spark, spark]");
- }
- }
- if (line.hasOption("explain")) {
- dmlOptions.explainType = ExplainType.RUNTIME;
- String explainType = line.getOptionValue("explain");
- if (explainType != null){
- if (explainType.equalsIgnoreCase("hops")) dmlOptions.explainType = ExplainType.HOPS;
- else if (explainType.equalsIgnoreCase("runtime")) dmlOptions.explainType = ExplainType.RUNTIME;
- else if (explainType.equalsIgnoreCase("recompile_hops")) dmlOptions.explainType = ExplainType.RECOMPILE_HOPS;
- else if (explainType.equalsIgnoreCase("recompile_runtime")) dmlOptions.explainType = ExplainType.RECOMPILE_RUNTIME;
- else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime]");
- }
- }
- dmlOptions.stats = line.hasOption("stats");
- if (dmlOptions.stats){
- String statsCount = line.getOptionValue("stats");
- if (statsCount != null) {
- try {
- dmlOptions.statsCount = Integer.parseInt(statsCount);
- } catch (NumberFormatException e) {
- throw new org.apache.commons.cli.ParseException("Invalid argument specified for -stats option, must be a valid integer");
- }
- }
- }
- dmlOptions.memStats = line.hasOption("mem");
-
- dmlOptions.clean = line.hasOption("clean");
-
- if (line.hasOption("config")){
- dmlOptions.configFile = line.getOptionValue("config");
- }
-
- if (line.hasOption("f")){
- dmlOptions.filePath = line.getOptionValue("f");
- }
-
- if (line.hasOption("s")){
- dmlOptions.script = line.getOptionValue("s");
- }
-
- // Positional arguments map is created as ("$1", "a"), ("$2", 123), ....
- if (line.hasOption("args")){
- String[] argValues = line.getOptionValues("args");
- for (int k=0; k<argValues.length; k++){
- String str = argValues[k];
- if (!str.isEmpty()) {
- dmlOptions.argVals.put("$" + (k+1), str);
- }
- }
- }
-
- // Named arguments map is created as ("$K, 123), ("$X", "X.csv"), ....
- if (line.hasOption("nvargs")){
- String varNameRegex = "^[a-zA-Z]([a-zA-Z0-9_])*$";
- String[] nvargValues = line.getOptionValues("nvargs");
- for (String str : nvargValues){
- if (!str.isEmpty()){
- String[] kv = str.split("=");
- if (kv.length != 2){
- throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, must be a list of space separated K=V pairs, where K is a valid name of a variable in the DML/PyDML program");
- }
- if (!kv[0].matches(varNameRegex)) {
- throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, " + kv[0] + " does not seem like a valid variable name in DML. Valid variable names in DML start with upper-case or lower-case letter, and contain only letters, digits, or underscores");
- }
- dmlOptions.argVals.put("$" + kv[0], kv[1]);
- }
- }
- }
-
- return dmlOptions;
-
- }
-
- /**
- * Creates an {@link Options} instance for the command line parameters
- * As of SystemML 0.13, Apache Commons CLI 1.2 is transitively in the classpath
- * However the most recent version of Apache Commons CLI is 1.4
- * Creating CLI options is done using Static methods. This obviously makes it
- * thread unsafe. Instead of {@link OptionBuilder}, CLI 1.4 uses Option.Builder which
- * has non-static methods.
- * @return an appropriate instance of {@link Options}
- */
- @SuppressWarnings("static-access")
- public static Options createCLIOptions() {
- Options options = new Options();
- Option nvargsOpt = OptionBuilder.withArgName("key=value")
- .withDescription("parameterizes DML script with named parameters of the form <key=value>; <key> should be a valid identifier in DML/PyDML")
- .hasArgs()
- .create("nvargs");
- Option argsOpt = OptionBuilder.withArgName("argN")
- .withDescription("specifies positional parameters; first value will replace $1 in DML program; $2 will replace 2nd and so on")
- .hasArgs()
- .create("args");
- Option configOpt = OptionBuilder.withArgName("filename")
- .withDescription("uses a given configuration file (can be on local/hdfs/gpfs; default values in SystemML-config.xml")
- .hasArg()
- .create("config");
- Option cleanOpt = OptionBuilder.withDescription("cleans up all SystemML working directories (FS, DFS); all other flags are ignored in this mode. \n")
- .create("clean");
- Option statsOpt = OptionBuilder.withArgName("count")
- .withDescription("monitors and reports summary execution statistics; heavy hitter <count> is 10 unless overridden; default off")
- .hasOptionalArg()
- .create("stats");
- Option memOpt = OptionBuilder.withDescription("monitors and reports max memory consumption in CP; default off")
- .create("mem");
- Option explainOpt = OptionBuilder.withArgName("level")
- .withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime'")
- .hasOptionalArg()
- .create("explain");
- Option execOpt = OptionBuilder.withArgName("mode")
- .withDescription("sets execution mode; can be 'hadoop' / 'singlenode' / 'hybrid'[default] / 'hybrid_spark' / 'spark'")
- .hasArg()
- .create("exec");
- Option gpuOpt = OptionBuilder.withArgName("force")
- .withDescription("uses CUDA instructions when reasonable; set <force> option to skip conservative memory estimates and use GPU wherever possible; default off")
- .hasOptionalArg()
- .create("gpu");
- Option debugOpt = OptionBuilder.withDescription("runs in debug mode; default off")
- .create("debug");
- Option pythonOpt = OptionBuilder.withDescription("parses Python-like DML")
- .create("python");
- Option fileOpt = OptionBuilder.withArgName("filename")
- .withDescription("specifies dml/pydml file to execute; path can be local/hdfs/gpfs (prefixed with appropriate URI)")
- .isRequired()
- .hasArg()
- .create("f");
- Option scriptOpt = OptionBuilder.withArgName("script_contents")
- .withDescription("specified script string to execute directly")
- .isRequired()
- .hasArg()
- .create("s");
- Option helpOpt = OptionBuilder.withDescription("shows usage message")
- .create("help");
-
- OptionGroup fileOrScriptOpt = new OptionGroup();
- // Either a clean(-clean), a file(-f), a script(-s) or help(-help) needs to be specified
- fileOrScriptOpt.addOption(scriptOpt);
- fileOrScriptOpt.addOption(fileOpt);
- fileOrScriptOpt.addOption(cleanOpt);
- fileOrScriptOpt.addOption(helpOpt);
- fileOrScriptOpt.setRequired(true);
-
- OptionGroup argsOrNVArgsOpt = new OptionGroup();
- argsOrNVArgsOpt.addOption(nvargsOpt).addOption(argsOpt); // Either -args or -nvargs
-
- options.addOption(configOpt);
- options.addOption(cleanOpt);
- options.addOption(statsOpt);
- options.addOption(memOpt);
- options.addOption(explainOpt);
- options.addOption(execOpt);
- options.addOption(gpuOpt);
- options.addOption(debugOpt);
- options.addOption(pythonOpt);
- options.addOptionGroup(fileOrScriptOpt);
- options.addOptionGroup(argsOrNVArgsOpt);
- options.addOption(helpOpt);
- return options;
- }
-
- /**
* Single entry point for all public invocation alternatives (e.g.,
* main, executeScript, JaqlUdf etc)
*
@@ -461,16 +209,18 @@ public class DMLScript
* @param args arguments
* @return true if success, false otherwise
*/
+ @SuppressWarnings("null")
public static boolean executeScript( Configuration conf, String[] args ) {
//parse arguments and set execution properties
RUNTIME_PLATFORM oldrtplatform = rtplatform; //keep old rtplatform
ExplainType oldexplain = EXPLAIN; //keep old explain
- Options options = createCLIOptions();
+ DMLOptions dmlOptions = null;
+
try
{
- DMLOptions dmlOptions = parseCLArguments(args, options);
-
+ dmlOptions = DMLOptions.parseCLArguments(args);
+
STATISTICS = dmlOptions.stats;
STATISTICS_COUNT = dmlOptions.statsCount;
JMLC_MEM_STATISTICS = dmlOptions.memStats;
@@ -489,7 +239,7 @@ public class DMLScript
if (help) {
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp( "systemml", options );
+ formatter.printHelp( "systemml", dmlOptions.options );
return true;
}
@@ -525,13 +275,13 @@ public class DMLScript
catch(AlreadySelectedException e) {
System.err.println("Mutually exclusive options were selected. " + e.getMessage());
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp( "systemml", options );
+ formatter.printHelp( "systemml", dmlOptions.options );
return false;
}
catch(org.apache.commons.cli.ParseException e) {
System.err.println(e.getMessage());
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp( "systemml", options );
+ formatter.printHelp( "systemml", dmlOptions.options );
}
catch (ParseException | DMLScriptException e) {
throw e;
http://git-wip-us.apache.org/repos/asf/systemml/blob/5aadb4b2/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
index f3303de..00fc096 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
@@ -26,7 +26,7 @@ import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.api.DMLScript.DMLOptions;
+import org.apache.sysml.api.DMLOptions;
import org.apache.sysml.api.DMLScript.EvictionPolicy;
import org.apache.sysml.api.ScriptExecutorUtils;
import org.apache.sysml.api.jmlc.JMLCUtils;
http://git-wip-us.apache.org/repos/asf/systemml/blob/5aadb4b2/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java b/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java
index 8018bcf..95b671a 100644
--- a/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java
+++ b/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java
@@ -1,18 +1,18 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
+ * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
+ * regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
+ * KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
@@ -23,8 +23,8 @@ import java.util.Map;
import org.apache.commons.cli.AlreadySelectedException;
import org.apache.commons.cli.MissingOptionException;
-import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.sysml.api.DMLOptions;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.mlcontext.ScriptType;
import org.apache.sysml.utils.Explain;
@@ -34,382 +34,342 @@ import org.junit.Test;
public class CLIOptionsParserTest {
- @Test(expected = MissingOptionException.class)
- public void testNoOptions() throws Exception {
- String cl = "systemml";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- @Test
- public void testFile() throws Exception {
- String cl = "systemml -f test.dml";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals("test.dml", o.filePath);
- Assert.assertEquals(ScriptType.DML, o.scriptType);
-
- }
-
- @Test
- public void testScript() throws Exception {
- String cl = "systemml -s \"print('hello')\"";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals("print('hello')", o.script);
- }
-
- @Test
- public void testConfig() throws Exception {
- String cl = "systemml -s \"print('hello')\" -config SystemML-config.xml";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals("print('hello')", o.script);
- Assert.assertEquals("SystemML-config.xml", o.configFile);
- }
-
- @Test
- public void testDebug() throws Exception {
- String cl = "systemml -s \"print('hello')\" -debug";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals("print('hello')", o.script);
- Assert.assertEquals(true, o.debug);
- }
-
- @Test
- public void testClean() throws Exception {
- String cl = "systemml -clean";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(true, o.clean);
- }
-
- @Test(expected = AlreadySelectedException.class)
- public void testBadClean() throws Exception {
- String cl = "systemml -clean -f test.dml";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- @Test(expected = AlreadySelectedException.class)
- public void testBadScript() throws Exception {
- String cl = "systemml -f test.dml -s \"print('hello')\"";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- @Test
- public void testStats() throws Exception {
- String cl = "systemml -f test.dml -stats";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(true, o.stats);
- Assert.assertEquals(10, o.statsCount);
- }
-
- @Test
- public void testStatsCount() throws Exception {
- String cl = "systemml -f test.dml -stats 9123";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(true, o.stats);
- Assert.assertEquals(9123, o.statsCount);
- }
-
- @Test(expected = ParseException.class)
- public void testBadStats() throws Exception {
- String cl = "systemml -f test.dml -stats help";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(true, o.stats);
- }
-
- @Test
- public void testGPUForce() throws Exception {
- String cl = "systemml -f test.dml -gpu force";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(true, o.gpu);
- Assert.assertEquals(true, o.forceGPU);
- }
-
- @Test(expected = ParseException.class)
- public void testBadGPUOption() throws Exception {
- String cl = "systemml -f test.dml -gpu f2orce";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- @Test
- public void testPython() throws Exception {
- String cl = "systemml -f test.dml -python";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(ScriptType.PYDML, o.scriptType);
- }
-
- @Test
- public void testHelp() throws Exception {
- String cl = "systemml -help";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(true, o.help);
- }
-
- @Test(expected = AlreadySelectedException.class)
- public void testBadHelp() throws Exception {
- String cl = "systemml -help -clean";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(true, o.help);
- }
-
- @Test
- public void testExplain1() throws Exception {
- String cl = "systemml -f test.dml -explain";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(Explain.ExplainType.RUNTIME, o.explainType);
- }
-
- @Test
- public void testExplain2() throws Exception {
- String cl = "systemml -f test.dml -explain hops";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(Explain.ExplainType.HOPS, o.explainType);
- }
-
- @Test
- public void testExplain3() throws Exception {
- String cl = "systemml -f test.dml -explain runtime";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(Explain.ExplainType.RUNTIME, o.explainType);
- }
-
- @Test
- public void testExplain4() throws Exception {
- String cl = "systemml -f test.dml -explain recompile_hops";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(Explain.ExplainType.RECOMPILE_HOPS, o.explainType);
- }
-
- @Test
- public void testExplain5() throws Exception {
- String cl = "systemml -f test.dml -explain recompile_runtime";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(Explain.ExplainType.RECOMPILE_RUNTIME, o.explainType);
- }
-
- @Test
- public void testExec1() throws Exception {
- String cl = "systemml -f test.dml -exec hadoop";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HADOOP, o.execMode);
- }
-
- @Test
- public void testExec2() throws Exception {
- String cl = "systemml -f test.dml -exec spark";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.SPARK, o.execMode);
- }
-
- @Test
- public void testExec3() throws Exception {
- String cl = "systemml -f test.dml -exec singlenode";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.SINGLE_NODE, o.execMode);
- }
-
- @Test
- public void testExec4() throws Exception {
- String cl = "systemml -f test.dml -exec hybrid";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HYBRID, o.execMode);
- }
-
- @Test
- public void testExec5() throws Exception {
- String cl = "systemml -f test.dml -exec hybrid_spark";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK, o.execMode);
- }
-
- @Test(expected = ParseException.class)
- public void testBadExec() throws Exception {
- String cl = "systemml -f test.dml -exec new_system";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- @Test
- public void testArgs1() throws Exception {
- String cl = "systemml -f test.dml -args 10 \"x.csv\"";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Map<String, String> m = o.argVals;
- Assert.assertEquals(2, m.size());
- Assert.assertEquals("10", m.get("$1"));
- Assert.assertEquals("x.csv", m.get("$2"));
- }
-
- @Test
- public void testArgs2() throws Exception {
- String cl = "systemml -f test.dml -args 10 \"x.csv\" 1234.2 systemml.conf -config systemml.conf";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Map<String, String> m = o.argVals;
- Assert.assertEquals(4, m.size());
- Assert.assertEquals("10", m.get("$1"));
- Assert.assertEquals("x.csv", m.get("$2"));
- Assert.assertEquals("1234.2", m.get("$3"));
- Assert.assertEquals("systemml.conf", m.get("$4"));
- }
-
- @Test(expected = ParseException.class)
- public void testBadArgs1() throws Exception {
- String cl = "systemml -f test.dml -args -config systemml.conf";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- @Test
- public void testNVArgs1() throws Exception {
- String cl = "systemml -f test.dml -nvargs A=12 B=x.csv my123=12.2";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Map<String, String> m = o.argVals;
- Assert.assertEquals(3, m.size());
- Assert.assertEquals("12", m.get("$A"));
- Assert.assertEquals("x.csv", m.get("$B"));
- Assert.assertEquals("12.2", m.get("$my123"));
- }
-
- @Test(expected = ParseException.class)
- public void testBadNVArgs1() throws Exception {
- String cl = "systemml -f test.dml -nvargs";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- @Test(expected = ParseException.class)
- public void testBadNVArgs2() throws Exception {
- String cl = "systemml -f test.dml -nvargs asd qwe";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- @Test(expected = ParseException.class)
- public void testBadNVArgs3() throws Exception {
- String cl = "systemml -f test.dml -nvargs $X=12";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- @Test(expected = ParseException.class)
- public void testBadNVArgs4() throws Exception {
- String cl = "systemml -f test.dml -nvargs 123=123";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.parseCLArguments(args, options);
- }
-
- /**
- * For Apache Commons CLI, if an argument to an option is enclosed in quotes,
- * the leading and trailing quotes are stripped away. For instance, if the options is -arg and the
- * argument is "foo"
- * -args "foo"
- * Commons CLI will strip the quotes from "foo". This becomes troublesome when you really do
- * want to pass in "foo" and not just foo.
- * A way around this is to use 'foo` as done in {@link CLIOptionsParserTest#testNVArgs3()}
- */
- @Test
- public void testNVArgs2() throws Exception {
- String cl = "systemml -f test.dml -args \"def\"";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Map<String, String> m = o.argVals;
- Assert.assertEquals("def", m.get("$1"));
- }
-
-
- /**
- * See comment in {@link CLIOptionsParserTest#testNVArgs2()}
- */
- @Test
- public void testNVArgs3() throws Exception {
- String cl = "systemml -f test.dml -args 'def'";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Map<String, String> m = o.argVals;
- Assert.assertEquals("'def'", m.get("$1"));
- }
-
- /**
- * See comment in {@link CLIOptionsParserTest#testNVArgs2()}
- * Additionally, if we try to pass something like
- * -nvargs X="foo"
- * Commons CLI will strip the leading and trailing quotes (viz. double quotes), which
- * causes it to return
- * X="foo
- * The way to overcome this is to enclose the <value> of the <key=value> pair in single quotes
- * and strip them away in the parsing code ourselves.
- * TODO: Read the javadoc for this method, we can add in this logic if required
- */
- @Test
- public void testNVArgs4() throws Exception {
- String cl = "systemml -f test.dml -nvargs abc='def'";
- String[] args = cl.split(" ");
- Options options = DMLScript.createCLIOptions();
- DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options);
- Map<String, String> m = o.argVals;
- Assert.assertEquals("'def'", m.get("$abc"));
- }
-
+ @Test(expected = MissingOptionException.class)
+ public void testNoOptions() throws Exception {
+ String cl = "systemml";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ @Test
+ public void testFile() throws Exception {
+ String cl = "systemml -f test.dml";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals("test.dml", o.filePath);
+ Assert.assertEquals(ScriptType.DML, o.scriptType);
+ }
+
+ @Test
+ public void testScript() throws Exception {
+ String cl = "systemml -s \"print('hello')\"";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals("print('hello')", o.script);
+ }
+
+ @Test
+ public void testConfig() throws Exception {
+ String cl = "systemml -s \"print('hello')\" -config SystemML-config.xml";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals("print('hello')", o.script);
+ Assert.assertEquals("SystemML-config.xml", o.configFile);
+ }
+
+ @Test
+ public void testDebug() throws Exception {
+ String cl = "systemml -s \"print('hello')\" -debug";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals("print('hello')", o.script);
+ Assert.assertEquals(true, o.debug);
+ }
+
+ @Test
+ public void testClean() throws Exception {
+ String cl = "systemml -clean";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(true, o.clean);
+ }
+
+ @Test(expected = AlreadySelectedException.class)
+ public void testBadClean() throws Exception {
+ String cl = "systemml -clean -f test.dml";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ @Test(expected = AlreadySelectedException.class)
+ public void testBadScript() throws Exception {
+ String cl = "systemml -f test.dml -s \"print('hello')\"";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ @Test
+ public void testStats() throws Exception {
+ String cl = "systemml -f test.dml -stats";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(true, o.stats);
+ Assert.assertEquals(10, o.statsCount);
+ }
+
+ @Test
+ public void testStatsCount() throws Exception {
+ String cl = "systemml -f test.dml -stats 9123";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(true, o.stats);
+ Assert.assertEquals(9123, o.statsCount);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testBadStats() throws Exception {
+ String cl = "systemml -f test.dml -stats help";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(true, o.stats);
+ }
+
+ @Test
+ public void testGPUForce() throws Exception {
+ String cl = "systemml -f test.dml -gpu force";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(true, o.gpu);
+ Assert.assertEquals(true, o.forceGPU);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testBadGPUOption() throws Exception {
+ String cl = "systemml -f test.dml -gpu f2orce";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ @Test
+ public void testPython() throws Exception {
+ String cl = "systemml -f test.dml -python";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(ScriptType.PYDML, o.scriptType);
+ }
+
+ @Test
+ public void testHelp() throws Exception {
+ String cl = "systemml -help";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(true, o.help);
+ }
+
+ @Test(expected = AlreadySelectedException.class)
+ public void testBadHelp() throws Exception {
+ String cl = "systemml -help -clean";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(true, o.help);
+ }
+
+ @Test
+ public void testExplain1() throws Exception {
+ String cl = "systemml -f test.dml -explain";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(Explain.ExplainType.RUNTIME, o.explainType);
+ }
+
+ @Test
+ public void testExplain2() throws Exception {
+ String cl = "systemml -f test.dml -explain hops";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(Explain.ExplainType.HOPS, o.explainType);
+ }
+
+ @Test
+ public void testExplain3() throws Exception {
+ String cl = "systemml -f test.dml -explain runtime";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(Explain.ExplainType.RUNTIME, o.explainType);
+ }
+
+ @Test
+ public void testExplain4() throws Exception {
+ String cl = "systemml -f test.dml -explain recompile_hops";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(Explain.ExplainType.RECOMPILE_HOPS, o.explainType);
+ }
+
+ @Test
+ public void testExplain5() throws Exception {
+ String cl = "systemml -f test.dml -explain recompile_runtime";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(Explain.ExplainType.RECOMPILE_RUNTIME, o.explainType);
+ }
+
+ @Test
+ public void testExec1() throws Exception {
+ String cl = "systemml -f test.dml -exec hadoop";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HADOOP, o.execMode);
+ }
+
+ @Test
+ public void testExec2() throws Exception {
+ String cl = "systemml -f test.dml -exec spark";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.SPARK, o.execMode);
+ }
+
+ @Test
+ public void testExec3() throws Exception {
+ String cl = "systemml -f test.dml -exec singlenode";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.SINGLE_NODE, o.execMode);
+ }
+
+ @Test
+ public void testExec4() throws Exception {
+ String cl = "systemml -f test.dml -exec hybrid";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HYBRID, o.execMode);
+ }
+
+ @Test
+ public void testExec5() throws Exception {
+ String cl = "systemml -f test.dml -exec hybrid_spark";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK, o.execMode);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testBadExec() throws Exception {
+ String cl = "systemml -f test.dml -exec new_system";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ @Test
+ public void testArgs1() throws Exception {
+ String cl = "systemml -f test.dml -args 10 \"x.csv\"";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Map<String, String> m = o.argVals;
+ Assert.assertEquals(2, m.size());
+ Assert.assertEquals("10", m.get("$1"));
+ Assert.assertEquals("x.csv", m.get("$2"));
+ }
+
+ @Test
+ public void testArgs2() throws Exception {
+ String cl = "systemml -f test.dml -args 10 \"x.csv\" 1234.2 systemml.conf -config systemml.conf";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Map<String, String> m = o.argVals;
+ Assert.assertEquals(4, m.size());
+ Assert.assertEquals("10", m.get("$1"));
+ Assert.assertEquals("x.csv", m.get("$2"));
+ Assert.assertEquals("1234.2", m.get("$3"));
+ Assert.assertEquals("systemml.conf", m.get("$4"));
+ }
+
+ @Test(expected = ParseException.class)
+ public void testBadArgs1() throws Exception {
+ String cl = "systemml -f test.dml -args -config systemml.conf";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ @Test
+ public void testNVArgs1() throws Exception {
+ String cl = "systemml -f test.dml -nvargs A=12 B=x.csv my123=12.2";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Map<String, String> m = o.argVals;
+ Assert.assertEquals(3, m.size());
+ Assert.assertEquals("12", m.get("$A"));
+ Assert.assertEquals("x.csv", m.get("$B"));
+ Assert.assertEquals("12.2", m.get("$my123"));
+ }
+
+ @Test(expected = ParseException.class)
+ public void testBadNVArgs1() throws Exception {
+ String cl = "systemml -f test.dml -nvargs";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testBadNVArgs2() throws Exception {
+ String cl = "systemml -f test.dml -nvargs asd qwe";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testBadNVArgs3() throws Exception {
+ String cl = "systemml -f test.dml -nvargs $X=12";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testBadNVArgs4() throws Exception {
+ String cl = "systemml -f test.dml -nvargs 123=123";
+ String[] args = cl.split(" ");
+ DMLOptions.parseCLArguments(args);
+ }
+
+ /**
+ * For Apache Commons CLI, if an argument to an option is enclosed in quotes,
+ * the leading and trailing quotes are stripped away. For instance, if the options is -arg and the
+ * argument is "foo"
+ * -args "foo"
+ * Commons CLI will strip the quotes from "foo". This becomes troublesome when you really do
+ * want to pass in "foo" and not just foo.
+ * A way around this is to use 'foo` as done in {@link CLIOptionsParserTest#testNVArgs3()}
+ */
+ @Test
+ public void testNVArgs2() throws Exception {
+ String cl = "systemml -f test.dml -args \"def\"";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Map<String, String> m = o.argVals;
+ Assert.assertEquals("def", m.get("$1"));
+ }
+
+
+ /**
+ * See comment in {@link CLIOptionsParserTest#testNVArgs2()}
+ */
+ @Test
+ public void testNVArgs3() throws Exception {
+ String cl = "systemml -f test.dml -args 'def'";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Map<String, String> m = o.argVals;
+ Assert.assertEquals("'def'", m.get("$1"));
+ }
+
+ /**
+ * See comment in {@link CLIOptionsParserTest#testNVArgs2()}
+ * Additionally, if we try to pass something like
+ * -nvargs X="foo"
+ * Commons CLI will strip the leading and trailing quotes (viz. double quotes), which
+ * causes it to return
+ * X="foo
+ * The way to overcome this is to enclose the <value> of the <key=value> pair in single quotes
+ * and strip them away in the parsing code ourselves.
+ * TODO: Read the javadoc for this method, we can add in this logic if required
+ */
+ @Test
+ public void testNVArgs4() throws Exception {
+ String cl = "systemml -f test.dml -nvargs abc='def'";
+ String[] args = cl.split(" ");
+ DMLOptions o = DMLOptions.parseCLArguments(args);
+ Map<String, String> m = o.argVals;
+ Assert.assertEquals("'def'", m.get("$abc"));
+ }
}
\ No newline at end of file