You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/07/15 03:48:11 UTC
[1/2] systemml git commit: [SYSTEMML-1773] JMLC error handling
invalid argument/variable names
Repository: systemml
Updated Branches:
refs/heads/master 6778a63b0 -> 1b3dff06b
[SYSTEMML-1773] JMLC error handling invalid argument/variable names
This patch improves the error handling of invalid argument and variable
names passed to JMLC prepared scripts. Input/variables refer to the
left-hand-side of read assignments, while arguments (aka $ parameters)
can only be used on the right hand-side (potentially used in ifdefs). We
now explicitly check that variables do not start with $, while arguments
have to start with $. This helps to avoid silent errors, where passed
arguments with invalid names are simply ignored.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/4ca4d34f
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/4ca4d34f
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/4ca4d34f
Branch: refs/heads/master
Commit: 4ca4d34fe77b6c0726a2c7b06fbe069e956cdc25
Parents: 6778a63
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Jul 14 19:20:42 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Jul 14 20:49:22 2017 -0700
----------------------------------------------------------------------
.../org/apache/sysml/api/jmlc/Connection.java | 27 +++++++++++++++++---
1 file changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/4ca4d34f/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 7739a35..be440c8 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -25,9 +25,13 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.sysml.api.DMLException;
@@ -43,6 +47,7 @@ import org.apache.sysml.hops.rewrite.RewriteRemovePersistentReadWrite;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.LanguageException;
import org.apache.sysml.parser.ParseException;
import org.apache.sysml.parser.ParserFactory;
import org.apache.sysml.parser.ParserWrapper;
@@ -176,12 +181,21 @@ public class Connection implements Closeable
{
DMLScript.SCRIPT_TYPE = parsePyDML ? ScriptType.PYDML : ScriptType.DML;
- //prepare arguments
+ //check for valid names of passed arguments
+ String[] invalidArgs = args.keySet().stream()
+ .filter(k -> k==null || !k.startsWith("$")).toArray(String[]::new);
+ if( invalidArgs.length > 0 )
+ throw new LanguageException("Invalid argument names: "+Arrays.toString(invalidArgs));
+
+ //check for valid names of input and output variables
+ String[] invalidVars = asSet(inputs, outputs).stream()
+ .filter(k -> k==null || k.startsWith("$")).toArray(String[]::new);
+ if( invalidVars.length > 0 )
+ throw new LanguageException("Invalid variable names: "+Arrays.toString(invalidVars));
//simplified compilation chain
Program rtprog = null;
- try
- {
+ try {
//parsing
ParserWrapper parser = ParserFactory.createParser(parsePyDML ? ScriptType.PYDML : ScriptType.DML);
DMLProgram prog = parser.parse(null, script, args);
@@ -828,4 +842,11 @@ public class Connection implements Closeable
public FrameBlock readTransformMetaDataFromPath(String spec, String metapath, String colDelim) throws IOException {
return TfMetaUtils.readTransformMetaDataFromPath(spec, metapath, colDelim);
}
+
+ private Set<String> asSet(String[] inputs, String[] outputs) {
+ Set<String> ret = new HashSet<String>();
+ CollectionUtils.addAll(ret, inputs);
+ CollectionUtils.addAll(ret, outputs);
+ return ret;
+ }
}
[2/2] systemml git commit: [SYSTEMML-1772] Fix matrix mult exec type
selection w/ sparse mmchain
Posted by mb...@apache.org.
[SYSTEMML-1772] Fix matrix mult exec type selection w/ sparse mmchain
This patch fixes OOM issues encountered on perftest MultiLogReg, 100M x
1K, sparse, which were due to incorrect execution type selection of
matrix multiplications. The details are covered in the JIRA, but at a
high-level, this issue occurs if the final matrix multiplication of an
mmchain pattern (t(X) %*% (w*(X%*%v))), i.e., t(X) fits into CP but X
does not. This can happen for tall and skinny sparse matrices where each
row requires a memory overhead for sparse rows. We now select SPARK/MR
for the entire pattern, which is useful because the first matrix
multiplication will end-up in SPARK/MR anyway, allowing us to still
compile the fused mmchain operator and avoid unnecessary transfer
between CP and SPARK/MR backends.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/1b3dff06
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/1b3dff06
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/1b3dff06
Branch: refs/heads/master
Commit: 1b3dff06b8416975ed69bad1c119e0b33c3a6e95
Parents: 4ca4d34
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Jul 14 19:39:02 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Jul 14 20:49:23 2017 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/AggBinaryOp.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/1b3dff06/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
index 9077976..4f709b4 100644
--- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
@@ -458,6 +458,12 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
_etype = REMOTE;
}
+ //check for valid CP mmchain, send invalid memory requirements to remote
+ if( _etype == ExecType.CP && checkMapMultChain() != ChainType.NONE
+ && OptimizerUtils.getLocalMemBudget() <
+ getInput().get(0).getInput().get(0).getOutputMemEstimate() )
+ _etype = REMOTE;
+
//check for valid CP dimensions and matrix size
checkAndSetInvalidCPDimsAndSize();
}