You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/11/19 21:47:17 UTC
[35/50] [abbrv] incubator-systemml git commit: Fix rewrite 'split hop
dags' (invalid reuse of twrites), incl tests
Fix rewrite 'split hop dags' (invalid reuse of twrites), incl tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/26c2ce56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/26c2ce56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/26c2ce56
Branch: refs/heads/master
Commit: 26c2ce56db275a4620a20bc433f7f957738853ee
Parents: 732d7d2
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Wed Nov 4 18:56:11 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Wed Nov 4 18:56:11 2015 -0800
----------------------------------------------------------------------
.../RewriteSplitDagDataDependentOperators.java | 12 ++++--
.../recompile/RemoveEmptyPotpourriTest.java | 33 ++++++++++-------
.../recompile/remove_empty_potpourri4.R | 38 +++++++++++++++++++
.../recompile/remove_empty_potpourri4.dml | 39 ++++++++++++++++++++
4 files changed, 104 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/26c2ce56/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagDataDependentOperators.java b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
index e5b8792..2920853 100644
--- a/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
+++ b/src/main/java/com/ibm/bi/dml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
@@ -71,7 +71,7 @@ public class RewriteSplitDagDataDependentOperators extends StatementBlockRewrite
throws HopsException
{
ArrayList<StatementBlock> ret = new ArrayList<StatementBlock>();
-
+
//collect all unknown csv reads hops
ArrayList<Hop> cand = new ArrayList<Hop>();
collectDataDependentOperators( sb.get_hops(), cand );
@@ -98,9 +98,13 @@ public class RewriteSplitDagDataDependentOperators extends StatementBlockRewrite
ArrayList<Hop> sb1hops = new ArrayList<Hop>();
for( Hop c : cand )
{
- //if there are already transient writes use them and don't introduce artificial variables
+ //if there are already transient writes use them and don't introduce artificial variables;
+ //unless there are transient reads w/ the same variable name in the current dag which can
+ //lead to invalid reordering if variable consumers are not feeding into the candidate op.
boolean hasTWrites = hasTransientWriteParents(c);
-
+ boolean moveTWrite = hasTWrites ? HopRewriteUtils.rHasSimpleReadChain(c,
+ getFirstTransientWriteParent(c).getName()) : false;
+
String varname = null;
long rlen = c.getDim1();
long clen = c.getDim2();
@@ -108,7 +112,7 @@ public class RewriteSplitDagDataDependentOperators extends StatementBlockRewrite
long brlen = c.getRowsInBlock();
long bclen = c.getColsInBlock();
- if( hasTWrites ) //reuse existing transient_write
+ if( hasTWrites && moveTWrite) //reuse existing transient_write
{
Hop twrite = getFirstTransientWriteParent(c);
varname = twrite.getName();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/26c2ce56/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java b/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java
index 3f86e53..03e9740 100644
--- a/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java
+++ b/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/RemoveEmptyPotpourriTest.java
@@ -39,6 +39,7 @@ public class RemoveEmptyPotpourriTest extends AutomatedTestBase
private final static String TEST_NAME1 = "remove_empty_potpourri1";
private final static String TEST_NAME2 = "remove_empty_potpourri2";
private final static String TEST_NAME3 = "remove_empty_potpourri3";
+ private final static String TEST_NAME4 = "remove_empty_potpourri4";
private final static String TEST_DIR = "functions/recompile/";
private final static double eps = 1e-10;
@@ -51,44 +52,48 @@ public class RemoveEmptyPotpourriTest extends AutomatedTestBase
addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_DIR, TEST_NAME1, new String[] { "R" }));
addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_DIR, TEST_NAME2, new String[] { "R" }));
addTestConfiguration(TEST_NAME3, new TestConfiguration(TEST_DIR, TEST_NAME3, new String[] { "R" }));
+ addTestConfiguration(TEST_NAME4, new TestConfiguration(TEST_DIR, TEST_NAME4, new String[] { "R" }));
}
@Test
- public void testRemoveEmptySequenceReshapeNoRewrite()
- {
+ public void testRemoveEmptySequenceReshapeNoRewrite() {
runRemoveEmptyTest(TEST_NAME1, false);
}
@Test
- public void testRemoveEmptySequenceReshapeRewrite()
- {
+ public void testRemoveEmptySequenceReshapeRewrite() {
runRemoveEmptyTest(TEST_NAME1, true);
}
@Test
- public void testRemoveEmptySumColSumNoRewrite()
- {
+ public void testRemoveEmptySumColSumNoRewrite() {
runRemoveEmptyTest(TEST_NAME2, false);
}
@Test
- public void testRemoveEmptySumColSumRewrite()
- {
+ public void testRemoveEmptySumColSumRewrite() {
runRemoveEmptyTest(TEST_NAME2, true);
}
@Test
- public void testRemoveEmptyComplexDagSplitNoRewrite()
- {
+ public void testRemoveEmptyComplexDagSplitNoRewrite() {
runRemoveEmptyTest(TEST_NAME3, false);
}
-
@Test
- public void testRemoveEmptyComplexDagSplitRewrite()
- {
+ public void testRemoveEmptyComplexDagSplitRewrite() {
runRemoveEmptyTest(TEST_NAME3, true);
}
+
+ @Test
+ public void testRemoveEmptyComplexDagSplit2NoRewrite() {
+ runRemoveEmptyTest(TEST_NAME4, false);
+ }
+
+ @Test
+ public void testRemoveEmptyComplexDagSplit2Rewrite() {
+ runRemoveEmptyTest(TEST_NAME4, true);
+ }
/**
*
@@ -106,7 +111,7 @@ public class RemoveEmptyPotpourriTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + TEST_NAME + ".dml";
//note: stats required for runtime check of rewrite
- programArgs = new String[]{"-args", HOME + OUTPUT_DIR + "R" };
+ programArgs = new String[]{"-explain","-args", HOME + OUTPUT_DIR + "R" };
fullRScriptName = HOME + TEST_NAME + ".R";
rCmd = "Rscript" + " " + fullRScriptName + " " + HOME + EXPECTED_DIR;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/26c2ce56/src/test/scripts/functions/recompile/remove_empty_potpourri4.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/recompile/remove_empty_potpourri4.R b/src/test/scripts/functions/recompile/remove_empty_potpourri4.R
new file mode 100644
index 0000000..e6a7ddc
--- /dev/null
+++ b/src/test/scripts/functions/recompile/remove_empty_potpourri4.R
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# (C) Copyright IBM Corp. 2010, 2015
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+
+library("Matrix")
+
+X = matrix(1, 1000, 3);
+B = matrix(1, 1000, 2);
+C = matrix(7, 1000, 1);
+D = matrix(3, 1000, 1);
+
+E = cbind(X [, 1 : 2], B) * ((C * (1 - D))%*%matrix(1,1,4));
+X = X * C%*%matrix(1,1,3);
+n = nrow (X);
+
+R = X + sum(E) + n;
+
+cat(sum(R))
+
+writeMM(as(R, "CsparseMatrix"), paste(args[1], "R", sep=""));
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/26c2ce56/src/test/scripts/functions/recompile/remove_empty_potpourri4.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/recompile/remove_empty_potpourri4.dml b/src/test/scripts/functions/recompile/remove_empty_potpourri4.dml
new file mode 100644
index 0000000..cde22ca
--- /dev/null
+++ b/src/test/scripts/functions/recompile/remove_empty_potpourri4.dml
@@ -0,0 +1,39 @@
+#-------------------------------------------------------------
+#
+# (C) Copyright IBM Corp. 2010, 2015
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#-------------------------------------------------------------
+
+
+X = matrix(1, rows=1000, cols=3);
+B = matrix(1, rows=1000, cols=2);
+C = matrix(7, rows=1000, cols=1);
+D = matrix(3, rows=1000, cols=1);
+
+if(1==1){}
+
+tmp = append(X [, 1 : 2], B) * (C * (1 - D));
+E = removeEmpty (target = tmp, margin = "rows");
+
+X = removeEmpty (target = X * C, margin = "rows");
+n = nrow (X);
+
+if(1==1){}
+
+R = X + sum(E) + n;
+
+print(sum(R))
+
+write(R, $1);
\ No newline at end of file