You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/03/05 23:12:26 UTC
[1/3] systemml git commit: [MINOR] Cleanup unnecessary indirections
in dml/pydml parsers
Repository: systemml
Updated Branches:
refs/heads/master cf556f207 -> 1f63b09cd
[MINOR] Cleanup unnecessary indirections in dml/pydml parsers
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/727e69e2
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/727e69e2
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/727e69e2
Branch: refs/heads/master
Commit: 727e69e2893fc376d721915dff4831261a6d90f8
Parents: cf556f2
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Mar 5 13:12:59 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Mar 5 15:12:45 2018 -0800
----------------------------------------------------------------------
.../parser/common/CommonSyntacticValidator.java | 56 ++++++--------------
.../sysml/parser/dml/DmlSyntacticValidator.java | 20 +++----
.../parser/pydml/PydmlSyntacticValidator.java | 21 +++-----
3 files changed, 31 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/727e69e2/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java
index 67fbaf0..cc67ebe 100644
--- a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java
@@ -573,63 +573,51 @@ public abstract class CommonSyntacticValidator {
*/
protected abstract Expression handleLanguageSpecificFunction(ParserRuleContext ctx, String functionName, ArrayList<ParameterExpression> paramExpressions);
- /** Checks for builtin functions and does Action 'f'.
- * <p>
- * Constructs the
- * appropriate {@link AssignmentStatement} from
- * {@link CommonSyntacticValidator#functionCallAssignmentStatementHelper(ParserRuleContext, Set, Set, Expression, StatementInfo, Token, Token, String, String, ArrayList, boolean)}
- * or Assign to {@link Expression} from
- * DmlSyntacticValidator's exitBuiltinFunctionExpression(BuiltinFunctionExpressionContext).
- * </p>
+ /** Creates a builtin function expression.
*
* @param ctx antlr rule context
* @param functionName Name of the builtin function
* @param paramExpressions Array of parameter names and values
- * @param f action to perform
- * @return true if a builtin function was found
+ * @return expression if found otherwise null
*/
- protected boolean buildForBuiltInFunction(ParserRuleContext ctx, String functionName, ArrayList<ParameterExpression> paramExpressions, Action f) {
+ protected Expression buildForBuiltInFunction(ParserRuleContext ctx, String functionName, ArrayList<ParameterExpression> paramExpressions) {
// In global namespace, so it can be a builtin function
// Double verification: verify passed function name is a (non-parameterized) built-in function.
try {
if (functions.contains(functionName)) {
// It is a user function definition (which takes precedence if name same as built-in)
- return false;
+ return null;
}
Expression lsf = handleLanguageSpecificFunction(ctx, functionName, paramExpressions);
- if (lsf != null){
+ if (lsf != null) {
setFileLineColumn(lsf, ctx);
- f.execute(lsf);
- return true;
+ return lsf;
}
BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(ctx, functionName, paramExpressions, currentFile);
- if (bife != null){
+ if (bife != null) {
// It is a builtin function
- f.execute(bife);
- return true;
+ return bife;
}
ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression
.getParamBuiltinFunctionExpression(ctx, functionName, paramExpressions, currentFile);
if (pbife != null){
// It is a parameterized builtin function
- f.execute(pbife);
- return true;
+ return pbife;
}
// built-in read, rand ...
DataExpression dbife = DataExpression.getDataExpression(ctx, functionName, paramExpressions, currentFile, errorListener);
if (dbife != null){
- f.execute(dbife);
- return true;
+ return dbife;
}
- } catch(Exception e) {
+ }
+ catch(Exception e) {
notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return true;
}
- return false;
+ return null;
}
@@ -670,13 +658,11 @@ public abstract class CommonSyntacticValidator {
// For builtin functions with LHS
if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && !functions.contains(functionName)){
- final DataIdentifier ftarget = target;
- Action f = new Action() {
- @Override public void execute(Expression e) { setAssignmentStatement(ctx, info , ftarget, e); }
- };
- boolean validBIF = buildForBuiltInFunction(ctx, functionName, paramExpression, f);
- if (validBIF)
+ Expression e = buildForBuiltInFunction(ctx, functionName, paramExpression);
+ if( e != null ) {
+ setAssignmentStatement(ctx, info, target, e);
return;
+ }
}
// handle user-defined functions
@@ -695,14 +681,6 @@ public abstract class CommonSyntacticValidator {
return functCall;
}
- /**
- * To allow for different actions in
- * {@link CommonSyntacticValidator#functionCallAssignmentStatementHelper(ParserRuleContext, Set, Set, Expression, StatementInfo, Token, Token, String, String, ArrayList, boolean)}
- */
- public static interface Action {
- public void execute(Expression e);
- }
-
protected void setMultiAssignmentStatement(ArrayList<DataIdentifier> target, Expression expression, ParserRuleContext ctx, StatementInfo info) {
info.stmt = new MultiAssignmentStatement(target, expression);
info.stmt.setCtxValuesAndFilename(ctx, currentFile);
http://git-wip-us.apache.org/repos/asf/systemml/blob/727e69e2/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
index 4a1d6c3..a5169fb 100644
--- a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
@@ -530,18 +530,14 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D
functionName = convertedSyntax.functionName;
paramExpression = convertedSyntax.paramExpression;
}
- final ExpressionInfo info = ctx.info;
- Action f = new Action() {
- @Override public void execute(Expression e) { info.expr = e; }
- };
// handle built-in functions
- boolean validBIF = buildForBuiltInFunction(ctx, functionName, paramExpression, f);
- if (validBIF)
+ ctx.info.expr = buildForBuiltInFunction(ctx, functionName, paramExpression);
+ if( ctx.info.expr != null )
return;
// handle user-defined functions
- info.expr = createFunctionCall(ctx, namespace, functionName, paramExpression);
+ ctx.info.expr = createFunctionCall(ctx, namespace, functionName, paramExpression);
}
@@ -583,13 +579,11 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D
}
if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE)) {
- final FunctionCallMultiAssignmentStatementContext fctx = ctx;
- Action f = new Action() {
- @Override public void execute(Expression e) { setMultiAssignmentStatement(targetList, e, fctx, fctx.info); }
- };
- boolean validBIF = buildForBuiltInFunction(ctx, functionName, paramExpression, f);
- if (validBIF)
+ Expression e = buildForBuiltInFunction(ctx, functionName, paramExpression);
+ if( e != null ) {
+ setMultiAssignmentStatement(targetList, e, ctx, ctx.info);
return;
+ }
}
// Override default namespace for imported non-built-in function
http://git-wip-us.apache.org/repos/asf/systemml/blob/727e69e2/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
index 858eca6..5060ebd 100644
--- a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
@@ -1144,19 +1144,14 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements
functionName = convertedSyntax.functionName;
paramExpression = convertedSyntax.paramExpression;
}
-
- final ExpressionInfo info = ctx.info;
- Action f = new Action() {
- @Override public void execute(Expression e) { info.expr = e; }
- };
//handle builtin functions
- boolean validBIF = buildForBuiltInFunction(ctx, functionName, paramExpression, f);
- if (validBIF)
+ ctx.info.expr= buildForBuiltInFunction(ctx, functionName, paramExpression);
+ if( ctx.info.expr != null )
return;
// handle user-defined functions
- info.expr = createFunctionCall(ctx, namespace, functionName, paramExpression);
+ ctx.info.expr = createFunctionCall(ctx, namespace, functionName, paramExpression);
}
@Override
@@ -1199,13 +1194,11 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements
}
if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE)) {
- final FunctionCallMultiAssignmentStatementContext fctx = ctx;
- Action f = new Action() {
- @Override public void execute(Expression e) { setMultiAssignmentStatement(targetList, e, fctx, fctx.info); }
- };
- boolean validBIF = buildForBuiltInFunction(ctx, functionName, paramExpression, f);
- if (validBIF)
+ Expression e = buildForBuiltInFunction(ctx, functionName, paramExpression);
+ if( e != null ) {
+ setMultiAssignmentStatement(targetList, e, ctx, ctx.info);
return;
+ }
}
// Override default namespace for imported non-built-in function
[2/3] systemml git commit: [SYSTEMML-1444] Simplify nn-lstm layer
with UDFs in expressions
Posted by mb...@apache.org.
[SYSTEMML-1444] Simplify nn-lstm layer with UDFs in expressions
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/63be18a8
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/63be18a8
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/63be18a8
Branch: refs/heads/master
Commit: 63be18a840f4b898c5e91aa2f3ff2efd02690a97
Parents: 727e69e
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Mar 5 15:07:18 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Mar 5 15:12:47 2018 -0800
----------------------------------------------------------------------
scripts/nn/layers/lstm.dml | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/63be18a8/scripts/nn/layers/lstm.dml
----------------------------------------------------------------------
diff --git a/scripts/nn/layers/lstm.dml b/scripts/nn/layers/lstm.dml
index 68d5a5a..44942d2 100644
--- a/scripts/nn/layers/lstm.dml
+++ b/scripts/nn/layers/lstm.dml
@@ -90,15 +90,12 @@ forward = function(matrix[double] X, matrix[double] W, matrix[double] b, int T,
X_t = X[,(t-1)*D+1:t*D] # shape (N, D)
input = cbind(X_t, out_prev) # shape (N, D+M)
ifog = input %*% W + b # input, forget, output, and g gates; shape (N, 4M)
- tmp = sigmoid::forward(ifog[,1:3*M]) # i,f,o gates squashed with sigmoid
- ifog[,1:3*M] = tmp
- tmp = tanh::forward(ifog[,3*M+1:4*M]) # g gate squashed with tanh
- ifog[,3*M+1:4*M] = tmp
+ ifog[,1:3*M] = sigmoid::forward(ifog[,1:3*M]) # i,f,o gates squashed with sigmoid
+ ifog[,3*M+1:4*M] = tanh::forward(ifog[,3*M+1:4*M]) # g gate squashed with tanh
# c_t = f*prev_c + i*g
c = ifog[,M+1:2*M]*c_prev + ifog[,1:M]*ifog[,3*M+1:4*M] # shape (N, M)
# out_t = o*tanh(c)
- tmp = tanh::forward(c)
- out_t = ifog[,2*M+1:3*M] * tmp # shape (N, M)
+ out_t = ifog[,2*M+1:3*M] * tanh::forward(c) # shape (N, M)
# store
if (return_sequences) {
@@ -202,10 +199,8 @@ backward = function(matrix[double] dout, matrix[double] dc,
o = ifog[,2*M+1:3*M] # output gate, shape (N, M)
g = ifog[,3*M+1:4*M] # g gate, shape (N, M)
- tmp = tanh::backward(dout_t, ct)
- dct = dct + o*tmp # shape (N, M)
- tmp = tanh::forward(ct)
- do = tmp * dout_t # output gate, shape (N, M)
+ dct = dct + o*tanh::backward(dout_t, ct) # shape (N, M)
+ do = tanh::forward(ct) * dout_t # output gate, shape (N, M)
df = c_prev * dct # forget gate, shape (N, M)
dc_prev = f * dct # shape (N, M)
di = g * dct # input gate, shape (N, M)
[3/3] systemml git commit: [SYSTEMML-1444] Fix function call hoisting
in nested control flow
Posted by mb...@apache.org.
[SYSTEMML-1444] Fix function call hoisting in nested control flow
The recent simplification of the nn-lstm layer with UDFs in expressions
revealed a hidden issue of function call hoisting in nested control
flow. Specifically, newly added statement blocks where not reassigned
and thus lost, which led to validation errors due to non-existing input
variables in statements that have been altered during hoisting.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/1f63b09c
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/1f63b09c
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/1f63b09c
Branch: refs/heads/master
Commit: 1f63b09cdd0498a94f6bf4e8dfefbfa3cc27d948
Parents: 63be18a
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Mar 5 15:12:32 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Mar 5 15:12:49 2018 -0800
----------------------------------------------------------------------
.../org/apache/sysml/parser/StatementBlock.java | 24 +++++++++++++++-----
1 file changed, 18 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/1f63b09c/src/main/java/org/apache/sysml/parser/StatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/StatementBlock.java b/src/main/java/org/apache/sysml/parser/StatementBlock.java
index f7901c1..04e4a34 100644
--- a/src/main/java/org/apache/sysml/parser/StatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/StatementBlock.java
@@ -412,31 +412,43 @@ public class StatementBlock extends LiveVariableAnalysis implements ParseInfo
if (current instanceof FunctionStatementBlock) {
FunctionStatementBlock fsb = (FunctionStatementBlock)current;
FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
+ ArrayList<StatementBlock> tmp = new ArrayList<>();
for (StatementBlock sb : fstmt.getBody())
- rHoistFunctionCallsFromExpressions(sb);
+ tmp.addAll(rHoistFunctionCallsFromExpressions(sb));
+ fstmt.setBody(tmp);
}
else if (current instanceof WhileStatementBlock) {
WhileStatementBlock wsb = (WhileStatementBlock) current;
WhileStatement wstmt = (WhileStatement)wsb.getStatement(0);
//TODO handle predicates
+ ArrayList<StatementBlock> tmp = new ArrayList<>();
for (StatementBlock sb : wstmt.getBody())
- rHoistFunctionCallsFromExpressions(sb);
+ tmp.addAll(rHoistFunctionCallsFromExpressions(sb));
+ wstmt.setBody(tmp);
}
else if (current instanceof IfStatementBlock) {
IfStatementBlock isb = (IfStatementBlock) current;
IfStatement istmt = (IfStatement)isb.getStatement(0);
//TODO handle predicates
+ ArrayList<StatementBlock> tmp = new ArrayList<>();
for (StatementBlock sb : istmt.getIfBody())
- rHoistFunctionCallsFromExpressions(sb);
- for (StatementBlock sb : istmt.getElseBody())
- rHoistFunctionCallsFromExpressions(sb);
+ tmp.addAll(rHoistFunctionCallsFromExpressions(sb));
+ istmt.setIfBody(tmp);
+ if( istmt.getElseBody() != null && !istmt.getElseBody().isEmpty() ) {
+ ArrayList<StatementBlock> tmp2 = new ArrayList<>();
+ for (StatementBlock sb : istmt.getElseBody())
+ tmp2.addAll(rHoistFunctionCallsFromExpressions(sb));
+ istmt.setElseBody(tmp2);
+ }
}
else if (current instanceof ForStatementBlock) { //incl parfor
ForStatementBlock fsb = (ForStatementBlock) current;
ForStatement fstmt = (ForStatement)fsb.getStatement(0);
//TODO handle predicates
+ ArrayList<StatementBlock> tmp = new ArrayList<>();
for (StatementBlock sb : fstmt.getBody())
- rHoistFunctionCallsFromExpressions(sb);
+ tmp.addAll(rHoistFunctionCallsFromExpressions(sb));
+ fstmt.setBody(tmp);
}
else { //generic (last-level)
ArrayList<Statement> tmp = new ArrayList<>();