You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/07/08 21:26:42 UTC
[1/2] systemml git commit: [SYSTEMML-2428] Fix IPA for function calls
w/ unknown partial binding
Repository: systemml
Updated Branches:
refs/heads/master 63a1e2ac5 -> a1eb7bced
[SYSTEMML-2428] Fix IPA for function calls w/ unknown partial binding
This patch fixes the robustness of IPA for handling function calls with
unknown statistics and partial output bindings (i.e., more function
output parameters than bound function call outputs).
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/e83ae653
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/e83ae653
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/e83ae653
Branch: refs/heads/master
Commit: e83ae65349d8f479f7bad60f551cd145527b6071
Parents: 63a1e2a
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Jul 8 12:54:39 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Jul 8 14:27:29 2018 -0700
----------------------------------------------------------------------
.../sysml/hops/ipa/InterProceduralAnalysis.java | 21 +++-----
.../functions/misc/FunctionPotpourriTest.java | 8 +++
.../functions/misc/FunPotpourriSubsetReturn.dml | 51 ++++++++++++++++++++
3 files changed, 66 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/e83ae653/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index a46e841..c00ea6c 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -634,28 +634,21 @@ public class InterProceduralAnalysis
}
}
- private static void extractFunctionCallUnknownReturnStatistics( FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callVars )
- {
+ private static void extractFunctionCallUnknownReturnStatistics(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callVars) {
ArrayList<DataIdentifier> foutputOps = fstmt.getOutputParams();
String[] outputVars = fop.getOutputVariableNames();
String fkey = fop.getFunctionKey();
-
- try
- {
- for( int i=0; i<foutputOps.size(); i++ )
- {
+ try {
+ //robustness for subset of bound output variables
+ int olen = Math.min(foutputOps.size(), outputVars.length);
+ for( int i=0; i<olen; i++ ) {
DataIdentifier di = foutputOps.get(i);
String pvarname = outputVars[i]; //name in calling program
-
if( di.getDataType()==DataType.MATRIX )
- {
- MatrixObject moOut = createOutputMatrix(-1, -1, -1);
- callVars.put(pvarname, moOut);
- }
+ callVars.put(pvarname, createOutputMatrix(-1, -1, -1));
}
}
- catch( Exception ex )
- {
+ catch( Exception ex ) {
throw new HopsException( "Failed to extract output statistics of function "+fkey+".", ex);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/e83ae653/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java
index bcf7c46..e7634fa 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java
@@ -32,6 +32,8 @@ public class FunctionPotpourriTest extends AutomatedTestBase
private final static String TEST_NAME2 = "FunPotpourriComments";
private final static String TEST_NAME3 = "FunPotpourriNoReturn2";
private final static String TEST_NAME4 = "FunPotpourriEval";
+ private final static String TEST_NAME5 = "FunPotpourriSubsetReturn";
+
private final static String TEST_DIR = "functions/misc/";
private final static String TEST_CLASS_DIR = TEST_DIR + FunctionPotpourriTest.class.getSimpleName() + "/";
@@ -43,6 +45,7 @@ public class FunctionPotpourriTest extends AutomatedTestBase
addTestConfiguration( TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }) );
addTestConfiguration( TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" }) );
addTestConfiguration( TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] { "R" }) );
}
@Test
@@ -65,6 +68,11 @@ public class FunctionPotpourriTest extends AutomatedTestBase
runFunctionTest( TEST_NAME4, false );
}
+ @Test
+ public void testFunctionSubsetReturn() {
+ runFunctionTest( TEST_NAME5, false );
+ }
+
private void runFunctionTest(String testName, boolean error) {
TestConfiguration config = getTestConfiguration(testName);
loadTestConfiguration(config);
http://git-wip-us.apache.org/repos/asf/systemml/blob/e83ae653/src/test/scripts/functions/misc/FunPotpourriSubsetReturn.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/FunPotpourriSubsetReturn.dml b/src/test/scripts/functions/misc/FunPotpourriSubsetReturn.dml
new file mode 100644
index 0000000..c2c4e9e
--- /dev/null
+++ b/src/test/scripts/functions/misc/FunPotpourriSubsetReturn.dml
@@ -0,0 +1,51 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+arima_residuals = function(Matrix[Double] weights, Matrix[Double] X, Integer p, Integer P, Integer q, Integer Q, Integer s, String solver) return (Matrix[Double] errs, Matrix[Double] combined_weights){
+ combined_weights = weights
+ if (p>0 & P>0)
+ combined_weights = rbind(combined_weights, matrix(weights[1:p,] %*% t(weights[p+1:p+P,]), rows=p*P, cols=1))
+ b = X[,2:ncol(X)]%*%combined_weights
+ errs = X[,1] - b
+}
+
+X = matrix(1, 1000, 1)
+p = 2
+d = 0
+q = 0
+P = 0
+D = 0
+Q = 0
+s = 0
+totparamcols = p+P+Q+q+p*P
+num_rows = nrow(X)
+
+if(num_rows <= d)
+ print("non-seasonal differencing order should be smaller than length of the time-series")
+if(num_rows <= s*D)
+ print("seasonal differencing order should be smaller than number of observations divided by length of season")
+
+Z = cbind (X[1:nrow(X),], matrix(0, nrow(X), totparamcols))
+weights = matrix("0.459982 0.673987", 2, 1)
+
+f1 = arima_residuals(weights, Z, p, P, q, Q, s, "")
+f2 = arima_residuals(weights, Z, p, P, q, Q, s, "")
+print("out: " + sum(f1-f2))
[2/2] systemml git commit: [SYSTEMML-2429] Fix IPA function inlining
pass w/ partial binding
Posted by mb...@apache.org.
[SYSTEMML-2429] Fix IPA function inlining pass w/ partial binding
This patch fixes a side effect between IPA function inlining and IPA
dead code elimination, where the dead code elimination failed due to a
function call graph corruption on partial inlining. Specifically such a
partial inlining happens when a subset of function calls does not bind
all output parameters in which case these calls are not inlined.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/a1eb7bce
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/a1eb7bce
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/a1eb7bce
Branch: refs/heads/master
Commit: a1eb7bcedef100ef66f35eada2054fcbff00f6bc
Parents: e83ae65
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Jul 8 14:05:23 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Jul 8 14:27:36 2018 -0700
----------------------------------------------------------------------
.../sysml/hops/ipa/IPAPassInlineFunctions.java | 8 ++-
.../sysml/hops/ipa/InterProceduralAnalysis.java | 2 +-
.../functions/misc/FunctionPotpourriTest.java | 8 ++-
.../misc/FunPotpourriSubsetReturnDead.dml | 51 ++++++++++++++++++++
4 files changed, 65 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/a1eb7bce/src/main/java/org/apache/sysml/hops/ipa/IPAPassInlineFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/IPAPassInlineFunctions.java b/src/main/java/org/apache/sysml/hops/ipa/IPAPassInlineFunctions.java
index 7706ed6..4c7998d 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/IPAPassInlineFunctions.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/IPAPassInlineFunctions.java
@@ -72,6 +72,7 @@ public class IPAPassInlineFunctions extends IPAPass
ArrayList<Hop> hops = fstmt.getBody().get(0).getHops();
List<FunctionOp> fcalls = fgraph.getFunctionCalls(fkey);
List<StatementBlock> fcallsSB = fgraph.getFunctionCallsSB(fkey);
+ boolean removedAll = true;
for(int i=0; i<fcalls.size(); i++) {
FunctionOp op = fcalls.get(i);
if( LOG.isDebugEnabled() )
@@ -79,8 +80,10 @@ public class IPAPassInlineFunctions extends IPAPass
//step 0: robustness for special cases
if( op.getInput().size() != fstmt.getInputParams().size()
- || op.getOutputVariableNames().length != fstmt.getOutputParams().size() )
+ || op.getOutputVariableNames().length != fstmt.getOutputParams().size() ) {
+ removedAll = false;
continue;
+ }
//step 1: deep copy hop dag
ArrayList<Hop> hops2 = Recompiler.deepCopyHopsDag(hops);
@@ -110,7 +113,8 @@ public class IPAPassInlineFunctions extends IPAPass
//update the function call graph to avoid repeated inlining
//(and thus op replication) on repeated IPA calls
- fgraph.removeFunctionCalls(fkey);
+ if( removedAll )
+ fgraph.removeFunctionCalls(fkey);
}
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/a1eb7bce/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index c00ea6c..85a433a 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -101,7 +101,7 @@ public class InterProceduralAnalysis
static {
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("org.apache.sysml.hops.ipa.InterProceduralAnalysis")
+ Logger.getLogger("org.apache.sysml.hops.ipa")
.setLevel((Level) Level.DEBUG);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/a1eb7bce/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java
index e7634fa..fda4c2f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionPotpourriTest.java
@@ -33,7 +33,7 @@ public class FunctionPotpourriTest extends AutomatedTestBase
private final static String TEST_NAME3 = "FunPotpourriNoReturn2";
private final static String TEST_NAME4 = "FunPotpourriEval";
private final static String TEST_NAME5 = "FunPotpourriSubsetReturn";
-
+ private final static String TEST_NAME6 = "FunPotpourriSubsetReturnDead";
private final static String TEST_DIR = "functions/misc/";
private final static String TEST_CLASS_DIR = TEST_DIR + FunctionPotpourriTest.class.getSimpleName() + "/";
@@ -46,6 +46,7 @@ public class FunctionPotpourriTest extends AutomatedTestBase
addTestConfiguration( TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" }) );
addTestConfiguration( TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" }) );
addTestConfiguration( TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] { "R" }) );
}
@Test
@@ -73,6 +74,11 @@ public class FunctionPotpourriTest extends AutomatedTestBase
runFunctionTest( TEST_NAME5, false );
}
+ @Test
+ public void testFunctionSubsetReturnDead() {
+ runFunctionTest( TEST_NAME6, false );
+ }
+
private void runFunctionTest(String testName, boolean error) {
TestConfiguration config = getTestConfiguration(testName);
loadTestConfiguration(config);
http://git-wip-us.apache.org/repos/asf/systemml/blob/a1eb7bce/src/test/scripts/functions/misc/FunPotpourriSubsetReturnDead.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/FunPotpourriSubsetReturnDead.dml b/src/test/scripts/functions/misc/FunPotpourriSubsetReturnDead.dml
new file mode 100644
index 0000000..c7ed6fd
--- /dev/null
+++ b/src/test/scripts/functions/misc/FunPotpourriSubsetReturnDead.dml
@@ -0,0 +1,51 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+arima_residuals = function(Matrix[Double] weights, Matrix[Double] X, Integer p, Integer P, Integer q, Integer Q, Integer s, String solver) return (Matrix[Double] errs, Matrix[Double] combined_weights){
+ combined_weights = weights
+ if (p>0 & P>0)
+ combined_weights = rbind(combined_weights, matrix(weights[1:p,] %*% t(weights[p+1:p+P,]), rows=p*P, cols=1))
+ b = X[,2:ncol(X)]%*%combined_weights
+ errs = X[,1] - b
+}
+
+X = matrix(1, 1000, 1)
+p = 2
+d = 0
+q = 0
+P = 0
+D = 0
+Q = 0
+s = 0
+totparamcols = p+P+Q+q+p*P
+num_rows = nrow(X)
+
+if(num_rows <= d)
+ print("non-seasonal differencing order should be smaller than length of the time-series")
+if(num_rows <= s*D)
+ print("seasonal differencing order should be smaller than number of observations divided by length of season")
+
+Z = cbind (X[1:nrow(X),], matrix(0, nrow(X), totparamcols))
+weights = matrix("0.459982 0.673987", 2, 1)
+
+f1 = arima_residuals(weights, Z, p, P, q, Q, s, "")
+f2 = arima_residuals(weights, Z, p, P, q, Q, s, "")
+print("out: " + sum(f1))