You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/01/29 18:15:27 UTC
[1/2] incubator-systemml git commit: Fix automatic vectorization of
left indexing chains (size updates)
Repository: incubator-systemml
Updated Branches:
refs/heads/master c912ca46a -> 5956a6125
Fix automatic vectorization of left indexing chains (size updates)
This patch fixes incorrect top-down size updates of vectorized left
indexing chains, which led to incorrectly applied dynamic rewrites for
removing left indexing operations and thus incorrect results / errors.
Additionally, this change includes the following:
1) New test cases for automatic vectorization of row/column left
indexing chains
3) Fix rbind/cbind size updates w/ unknowns on refresh size information
3) Fix debug logging setup inter-procedural analysis
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/0a2b5872
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/0a2b5872
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/0a2b5872
Branch: refs/heads/master
Commit: 0a2b5872245c3139866917bd6230e8d88e185ada
Parents: c912ca4
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Jan 28 14:33:07 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Fri Jan 29 09:13:32 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/sysml/hops/BinaryOp.java | 8 ++
.../sysml/hops/ipa/InterProceduralAnalysis.java | 2 +-
.../rewrite/RewriteIndexingVectorization.java | 17 +--
.../vect/LeftIndexingChainUpdateTest.java | 107 +++++++++++++++++++
.../functions/vect/VectorizeLixChainCol.R | 38 +++++++
.../functions/vect/VectorizeLixChainCol.dml | 30 ++++++
.../functions/vect/VectorizeLixChainRow.R | 38 +++++++
.../functions/vect/VectorizeLixChainRow.dml | 30 ++++++
.../functions/vect/ZPackageSuite.java | 3 +-
9 files changed, 265 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a2b5872/src/main/java/org/apache/sysml/hops/BinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/BinaryOp.java b/src/main/java/org/apache/sysml/hops/BinaryOp.java
index eb5ee5c..7fd0e97 100644
--- a/src/main/java/org/apache/sysml/hops/BinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/BinaryOp.java
@@ -1438,9 +1438,13 @@ public class BinaryOp extends Hop
//ensure both columns are known, otherwise dangerous underestimation due to +(-1)
if( input1.getDim2()>0 && input2.getDim2()>0 )
setDim2( input1.getDim2() + input2.getDim2() );
+ else
+ setDim2(-1);
//ensure both nnz are known, otherwise dangerous underestimation due to +(-1)
if( input1.getNnz()>0 && input2.getNnz()>0 )
setNnz( input1.getNnz() + input2.getNnz() );
+ else
+ setNnz(-1);
}
else if( op == OpOp2.RBIND )
{
@@ -1449,9 +1453,13 @@ public class BinaryOp extends Hop
//ensure both rows are known, otherwise dangerous underestimation due to +(-1)
if( input1.getDim1()>0 && input2.getDim1()>0 )
setDim1( input1.getDim1() + input2.getDim1() );
+ else
+ setDim1(-1);
//ensure both nnz are known, otherwise dangerous underestimation due to +(-1)
if( input1.getNnz()>0 && input2.getNnz()>0 )
setNnz( input1.getNnz() + input2.getNnz() );
+ else
+ setNnz(-1);
}
else if ( op == OpOp2.SOLVE )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a2b5872/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index 415aa80..8c6fe81 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -131,7 +131,7 @@ public class InterProceduralAnalysis
static {
// for internal debugging only
if( LDEBUG ) {
- Logger.getLogger("org.apache.sysml.parser.InterProceduralAnalysis")
+ Logger.getLogger("org.apache.sysml.hops.ipa.InterProceduralAnalysis")
.setLevel((Level) Level.DEBUG);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a2b5872/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java
index c50ad41..26aeb1b 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteIndexingVectorization.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.IndexingOp;
@@ -208,7 +207,7 @@ public class RewriteIndexingVectorization extends HopRewriteRule
@SuppressWarnings("unchecked")
private void vectorizeLeftIndexing( Hop hop )
throws HopsException
- {
+ {
if( hop instanceof LeftIndexingOp ) //left indexing
{
LeftIndexingOp ihop0 = (LeftIndexingOp) hop;
@@ -251,12 +250,14 @@ public class RewriteIndexingVectorization extends HopRewriteRule
HopRewriteUtils.removeChildReference(current, input); //input data
HopRewriteUtils.addChildReference(current, newRix, 0);
- //reset row index all candidates
- for( Hop c : ihops ) {
+ //reset row index all candidates and refresh sizes (bottom-up)
+ for( int i=ihops.size()-1; i>=0; i-- ) {
+ Hop c = ihops.get(i);
HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(2), 2); //row lower expr
HopRewriteUtils.addChildReference(c, new LiteralOp(1), 2);
HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(3), 3); //row upper expr
HopRewriteUtils.addChildReference(c, new LiteralOp(1), 3);
+ ((LeftIndexingOp)c).setRowLowerEqualsUpper(true);
c.refreshSizeInformation();
}
@@ -289,6 +290,7 @@ public class RewriteIndexingVectorization extends HopRewriteRule
if( isSingleRow && isSingleCol && !appliedRow )
{
+
//collect simple chains (w/o multiple consumers) of left indexing ops
ArrayList<Hop> ihops = new ArrayList<Hop>();
ihops.add(ihop0);
@@ -305,6 +307,7 @@ public class RewriteIndexingVectorization extends HopRewriteRule
ihops.add( tmp );
current = tmp;
}
+
//apply rewrite if found candidates
if( ihops.size() > 1 ){
Hop input = current.getInput().get(0);
@@ -321,12 +324,14 @@ public class RewriteIndexingVectorization extends HopRewriteRule
HopRewriteUtils.removeChildReference(current, input); //input data
HopRewriteUtils.addChildReference(current, newRix, 0);
- //reset row index all candidates
- for( Hop c : ihops ) {
+ //reset col index all candidates and refresh sizes (bottom-up)
+ for( int i=ihops.size()-1; i>=0; i-- ) {
+ Hop c = ihops.get(i);
HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(4), 4); //col lower expr
HopRewriteUtils.addChildReference(c, new LiteralOp(1), 4);
HopRewriteUtils.removeChildReferenceByPos(c, c.getInput().get(5), 5); //col upper expr
HopRewriteUtils.addChildReference(c, new LiteralOp(1), 5);
+ ((LeftIndexingOp)c).setColLowerEqualsUpper(true);
c.refreshSizeInformation();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a2b5872/src/test/java/org/apache/sysml/test/integration/functions/vect/LeftIndexingChainUpdateTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/vect/LeftIndexingChainUpdateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/vect/LeftIndexingChainUpdateTest.java
new file mode 100644
index 0000000..ceaa3bf
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/vect/LeftIndexingChainUpdateTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.vect;
+
+import java.util.HashMap;
+
+import org.junit.Test;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+/**
+ *
+ */
+public class LeftIndexingChainUpdateTest extends AutomatedTestBase
+{
+ private final static String TEST_DIR = "functions/vect/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + LeftIndexingChainUpdateTest.class.getSimpleName() + "/";
+
+ private final static String TEST_NAME1 = "VectorizeLixChainRow";
+ private final static String TEST_NAME2 = "VectorizeLixChainCol";
+
+ @Override
+ public void setUp() {
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"}));
+ addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"}));
+ }
+
+ @Test
+ public void testVectorizeLixRowPos() {
+ runVectorizationTest( TEST_NAME1, true );
+ }
+
+ @Test
+ public void testVectorizeLixRowNoRewrite() {
+ runVectorizationTest( TEST_NAME1, false );
+ }
+
+ @Test
+ public void testVectorizeLixColPos() {
+ runVectorizationTest( TEST_NAME2, true );
+ }
+
+ @Test
+ public void testVectorizeLixColNoRewrite() {
+ runVectorizationTest( TEST_NAME2, false );
+ }
+
+ /**
+ *
+ * @param cfc
+ * @param vt
+ */
+ private void runVectorizationTest( String testName, boolean rewrites )
+ {
+ String TEST_NAME = testName;
+ boolean flag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+
+ try
+ {
+ TestConfiguration config = getTestConfiguration(TEST_NAME);
+ loadTestConfiguration(config);
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-args", output("R") };
+
+ fullRScriptName = HOME + TEST_NAME + ".R";
+ rCmd = getRCmd(inputDir(), expectedDir());
+
+ //run tests
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare results
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
+ TestUtils.compareMatrices(dmlfile, rfile, 1e-14, "DML", "R");
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = flag;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a2b5872/src/test/scripts/functions/vect/VectorizeLixChainCol.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/vect/VectorizeLixChainCol.R b/src/test/scripts/functions/vect/VectorizeLixChainCol.R
new file mode 100644
index 0000000..0fa519a
--- /dev/null
+++ b/src/test/scripts/functions/vect/VectorizeLixChainCol.R
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+
+library("Matrix")
+
+R = matrix(0, 4, 2);
+
+for( i in 1:2 ) {
+ R[1,2] = 2;
+ R[2,2] = 3;
+ R[3,2] = 5;
+}
+
+writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep=""));
+
+
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a2b5872/src/test/scripts/functions/vect/VectorizeLixChainCol.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/vect/VectorizeLixChainCol.dml b/src/test/scripts/functions/vect/VectorizeLixChainCol.dml
new file mode 100644
index 0000000..4492440
--- /dev/null
+++ b/src/test/scripts/functions/vect/VectorizeLixChainCol.dml
@@ -0,0 +1,30 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+R = matrix(0, rows=4, cols=2);
+
+for( i in 1:2 ) {
+ R[1,2] = 2;
+ R[2,2] = 3;
+ R[3,2] = 5;
+}
+
+write(R, $1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a2b5872/src/test/scripts/functions/vect/VectorizeLixChainRow.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/vect/VectorizeLixChainRow.R b/src/test/scripts/functions/vect/VectorizeLixChainRow.R
new file mode 100644
index 0000000..501b0e4
--- /dev/null
+++ b/src/test/scripts/functions/vect/VectorizeLixChainRow.R
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+
+library("Matrix")
+
+R = matrix(0, 2, 4);
+
+for( i in 1:2 ) {
+ R[2,1] = 2;
+ R[2,2] = 3;
+ R[2,3] = 5;
+}
+
+writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep=""));
+
+
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a2b5872/src/test/scripts/functions/vect/VectorizeLixChainRow.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/vect/VectorizeLixChainRow.dml b/src/test/scripts/functions/vect/VectorizeLixChainRow.dml
new file mode 100644
index 0000000..413b075
--- /dev/null
+++ b/src/test/scripts/functions/vect/VectorizeLixChainRow.dml
@@ -0,0 +1,30 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+R = matrix(0, rows=2, cols=4);
+
+for( i in 1:2 ) {
+ R[2,1] = 2;
+ R[2,2] = 3;
+ R[2,3] = 5;
+}
+
+write(R, $1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a2b5872/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java
index 4eff5f0..11908dc 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/vect/ZPackageSuite.java
@@ -26,7 +26,8 @@ import org.junit.runners.Suite;
* won't run two of them at once. */
@RunWith(Suite.class)
@Suite.SuiteClasses({
- AutoVectorizationTest.class
+ AutoVectorizationTest.class,
+ LeftIndexingChainUpdateTest.class,
})
[2/2] incubator-systemml git commit: New static simplification
rewrite 'pushdown unary-aggregate transpose'
Posted by mb...@apache.org.
New static simplification rewrite 'pushdown unary-aggregate transpose'
This new static (size-independent) rewrite allows to push row/column
aggregates below a transpose operation. Examples are colSums(t(X)) ->
t(rowSums(X)) and rowSums(t(X)) -> t(colSums(X)), where we support the
following aggregate functions: sum, sum_sq, min, max, mean, var. The
benefits are (1) likely reduced input size to transpose, and (2)
potentially noops for vector transpose. This patch also includes the
relevant tests for a subset of aggregate functions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5956a612
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5956a612
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5956a612
Branch: refs/heads/master
Commit: 5956a61255ca4f0ffc1f3b1943dbb2f3fff969bc
Parents: 0a2b587
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Jan 28 21:03:25 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Fri Jan 29 09:13:39 2016 -0800
----------------------------------------------------------------------
.../RewriteAlgebraicSimplificationStatic.java | 55 ++++++-
.../functions/misc/RewritePushdownUaggTest.java | 157 +++++++++++++++++++
.../functions/misc/RewritePushdownColmins.R | 31 ++++
.../functions/misc/RewritePushdownColmins.dml | 26 +++
.../functions/misc/RewritePushdownColsums.R | 30 ++++
.../functions/misc/RewritePushdownColsums.dml | 26 +++
.../functions/misc/RewritePushdownRowmins.R | 31 ++++
.../functions/misc/RewritePushdownRowmins.dml | 26 +++
.../functions/misc/RewritePushdownRowsums.R | 30 ++++
.../functions/misc/RewritePushdownRowsums.dml | 26 +++
10 files changed, 435 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index 3aea9f0..2368153 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -59,13 +59,16 @@ import org.apache.sysml.parser.Expression.ValueType;
*
*/
public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
-{
-
+{
private static final Log LOG = LogFactory.getLog(RewriteAlgebraicSimplificationStatic.class.getName());
+ //valid aggregation operation types for rowOp to colOp conversions and vice versa
+ private static AggOp[] LOOKUP_VALID_ROW_COL_AGGREGATE = new AggOp[]{AggOp.SUM, AggOp.SUM_SQ, AggOp.MIN, AggOp.MAX, AggOp.MEAN, AggOp.VAR};
+
+ //valid binary operations for distributive and associate reorderings
private static OpOp2[] LOOKUP_VALID_DISTRIBUTIVE_BINARY = new OpOp2[]{OpOp2.PLUS, OpOp2.MINUS};
private static OpOp2[] LOOKUP_VALID_ASSOCIATIVE_BINARY = new OpOp2[]{OpOp2.PLUS, OpOp2.MULT};
-
+
@Override
public ArrayList<Hop> rewriteHopDAGs(ArrayList<Hop> roots, ProgramRewriteStatus state)
throws HopsException
@@ -139,6 +142,7 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
hi = simplifyDistributiveBinaryOperation(hop, hi, i);//e.g., (X-Y*X) -> (1-Y)*X
hi = simplifyBushyBinaryOperation(hop, hi, i); //e.g., (X*(Y*(Z%*%v))) -> (X*Y)*(Z%*%v)
hi = simplifyUnaryAggReorgOperation(hop, hi, i); //e.g., sum(t(X)) -> sum(X)
+ hi = pushdownUnaryAggTransposeOperation(hop, hi, i); //e.g., colSums(t(X)) -> t(rowSums(X))
hi = simplifyUnaryPPredOperation(hop, hi, i); //e.g., abs(ppred()) -> ppred(), others: round, ceil, floor
hi = simplifyTransposedAppend(hop, hi, i); //e.g., t(cbind(t(A),t(B))) -> rbind(A,B);
hi = fuseBinarySubDAGToUnaryOperation(hop, hi, i); //e.g., X*(1-X)-> sprop(X) || 1/(1+exp(-X)) -> sigmoid(X) || X*(X>0) -> selp(X)
@@ -819,6 +823,51 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
* @param pos
* @return
*/
+ private Hop pushdownUnaryAggTransposeOperation( Hop parent, Hop hi, int pos )
+ {
+ if( hi instanceof AggUnaryOp && hi.getParent().size()==1
+ && (((AggUnaryOp) hi).getDirection()==Direction.Row || ((AggUnaryOp) hi).getDirection()==Direction.Col)
+ && hi.getInput().get(0) instanceof ReorgOp && hi.getInput().get(0).getParent().size()==1
+ && ((ReorgOp)hi.getInput().get(0)).getOp()==ReOrgOp.TRANSPOSE
+ && HopRewriteUtils.isValidOp(((AggUnaryOp) hi).getOp(), LOOKUP_VALID_ROW_COL_AGGREGATE) )
+ {
+ AggUnaryOp uagg = (AggUnaryOp) hi;
+
+ //get input rewire existing operators (remove inner transpose)
+ Hop input = uagg.getInput().get(0).getInput().get(0);
+ HopRewriteUtils.removeAllChildReferences(hi.getInput().get(0));
+ HopRewriteUtils.removeAllChildReferences(hi);
+ HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
+
+ //pattern 1: row-aggregate to col aggregate, e.g., rowSums(t(X))->t(colSums(X))
+ if( uagg.getDirection()==Direction.Row ) {
+ uagg.setDirection(Direction.Col);
+ LOG.debug("Applied pushdownUnaryAggTransposeOperation1 (line "+hi.getBeginLine()+").");
+ }
+ //pattern 2: col-aggregate to row aggregate, e.g., colSums(t(X))->t(rowSums(X))
+ else if( uagg.getDirection()==Direction.Col ) {
+ uagg.setDirection(Direction.Row);
+ LOG.debug("Applied pushdownUnaryAggTransposeOperation2 (line "+hi.getBeginLine()+").");
+ }
+
+ //create outer transpose operation and rewire operators
+ HopRewriteUtils.addChildReference(uagg, input); uagg.refreshSizeInformation();
+ Hop trans = HopRewriteUtils.createTranspose(uagg); //incl refresh size
+ HopRewriteUtils.addChildReference(parent, trans, pos); //by def, same size
+
+ hi = trans;
+ }
+
+ return hi;
+ }
+
+ /**
+ *
+ * @param parent
+ * @param hi
+ * @param pos
+ * @return
+ */
private Hop simplifyUnaryPPredOperation( Hop parent, Hop hi, int pos )
{
if( hi instanceof UnaryOp && hi.getDataType()==DataType.MATRIX //unaryop
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/test/java/org/apache/sysml/test/integration/functions/misc/RewritePushdownUaggTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewritePushdownUaggTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewritePushdownUaggTest.java
new file mode 100644
index 0000000..348cfbb
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewritePushdownUaggTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.misc;
+
+import java.util.HashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
+
+/**
+ * Regression test for function recompile-once issue with literal replacement.
+ *
+ */
+public class RewritePushdownUaggTest extends AutomatedTestBase
+{
+ //two aggregation functions as examples
+ private static final String TEST_NAME1 = "RewritePushdownColsums";
+ private static final String TEST_NAME2 = "RewritePushdownRowsums";
+ private static final String TEST_NAME3 = "RewritePushdownColmins";
+ private static final String TEST_NAME4 = "RewritePushdownRowmins";
+
+ private static final String TEST_DIR = "functions/misc/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + RewritePushdownUaggTest.class.getSimpleName() + "/";
+
+ private static final int rows = 192;
+ private static final int cols = 293;
+ private static final double eps = Math.pow(10, -10);
+
+ @Override
+ public void setUp()
+ {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration( TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" }) );
+ }
+
+ @Test
+ public void testRewriteColSumsNoRewrite() {
+ testRewritePushdownUagg( TEST_NAME1, false );
+ }
+
+ @Test
+ public void testRewriteRowSumsNoRewrite() {
+ testRewritePushdownUagg( TEST_NAME2, false );
+ }
+
+ @Test
+ public void testRewriteColMinsNoRewrite() {
+ testRewritePushdownUagg( TEST_NAME3, false );
+ }
+
+ @Test
+ public void testRewriteRowMinsNoRewrite() {
+ testRewritePushdownUagg( TEST_NAME4, false );
+ }
+
+ @Test
+ public void testRewriteColSums() {
+ testRewritePushdownUagg( TEST_NAME1, true );
+ }
+
+ @Test
+ public void testRewriteRowSums() {
+ testRewritePushdownUagg( TEST_NAME2, true );
+ }
+
+ @Test
+ public void testRewriteColMins() {
+ testRewritePushdownUagg( TEST_NAME3, true );
+ }
+
+ @Test
+ public void testRewriteRowMins() {
+ testRewritePushdownUagg( TEST_NAME4, true );
+ }
+
+
+ /**
+ *
+ * @param condition
+ * @param branchRemoval
+ * @param IPA
+ */
+ private void testRewritePushdownUagg( String testname, boolean rewrites )
+ {
+ boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+
+ try
+ {
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testname + ".dml";
+ programArgs = new String[]{ "-stats","-args", input("X"), output("R") };
+
+ fullRScriptName = HOME + testname + ".R";
+ rCmd = getRCmd(inputDir(), expectedDir());
+
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
+
+ double[][] X = getRandomMatrix(rows, cols, -1, 1, 0.56d, 7);
+ writeInputMatrixWithMTD("X", X, true);
+
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare matrices
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
+ TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
+
+ //check matrix mult existence
+ String check = null;
+ if( testname.equals(TEST_NAME1) ) //colsums
+ check = rewrites ? "uark+" : "uack+";
+ else if( testname.equals(TEST_NAME2) ) //rowsums
+ check = rewrites ? "uack+" : "uark+";
+ else if( testname.equals(TEST_NAME3) ) //colmins
+ check = rewrites ? "uarmin" : "uacmin";
+ else if( testname.equals(TEST_NAME4) ) //rowmins
+ check = rewrites ? "uacmin" : "uarmin";
+
+ Assert.assertTrue( "Missing opcode: "+check, Statistics.getCPHeavyHitterOpCodes().contains(check) );
+ }
+ finally
+ {
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/test/scripts/functions/misc/RewritePushdownColmins.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/RewritePushdownColmins.R b/src/test/scripts/functions/misc/RewritePushdownColmins.R
new file mode 100644
index 0000000..4459646
--- /dev/null
+++ b/src/test/scripts/functions/misc/RewritePushdownColmins.R
@@ -0,0 +1,31 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+library("matrixStats")
+
+X = as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+R = t(as.matrix(colMins(t(X))));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/test/scripts/functions/misc/RewritePushdownColmins.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/RewritePushdownColmins.dml b/src/test/scripts/functions/misc/RewritePushdownColmins.dml
new file mode 100644
index 0000000..90d2668
--- /dev/null
+++ b/src/test/scripts/functions/misc/RewritePushdownColmins.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+X = read($1);
+R = colMins(t(X));
+
+write(R, $2);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/test/scripts/functions/misc/RewritePushdownColsums.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/RewritePushdownColsums.R b/src/test/scripts/functions/misc/RewritePushdownColsums.R
new file mode 100644
index 0000000..baafefe
--- /dev/null
+++ b/src/test/scripts/functions/misc/RewritePushdownColsums.R
@@ -0,0 +1,30 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X = as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+R = t(as.matrix(colSums(t(X))));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/test/scripts/functions/misc/RewritePushdownColsums.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/RewritePushdownColsums.dml b/src/test/scripts/functions/misc/RewritePushdownColsums.dml
new file mode 100644
index 0000000..2528da9
--- /dev/null
+++ b/src/test/scripts/functions/misc/RewritePushdownColsums.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+X = read($1);
+R = colSums(t(X));
+
+write(R, $2);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/test/scripts/functions/misc/RewritePushdownRowmins.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/RewritePushdownRowmins.R b/src/test/scripts/functions/misc/RewritePushdownRowmins.R
new file mode 100644
index 0000000..0c01834
--- /dev/null
+++ b/src/test/scripts/functions/misc/RewritePushdownRowmins.R
@@ -0,0 +1,31 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+library("matrixStats")
+
+X = as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+R = rowMins(t(X));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/test/scripts/functions/misc/RewritePushdownRowmins.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/RewritePushdownRowmins.dml b/src/test/scripts/functions/misc/RewritePushdownRowmins.dml
new file mode 100644
index 0000000..c5f80e5
--- /dev/null
+++ b/src/test/scripts/functions/misc/RewritePushdownRowmins.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+X = read($1);
+R = rowMins(t(X));
+
+write(R, $2);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/test/scripts/functions/misc/RewritePushdownRowsums.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/RewritePushdownRowsums.R b/src/test/scripts/functions/misc/RewritePushdownRowsums.R
new file mode 100644
index 0000000..2de9a23
--- /dev/null
+++ b/src/test/scripts/functions/misc/RewritePushdownRowsums.R
@@ -0,0 +1,30 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X = as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+R = rowSums(t(X));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5956a612/src/test/scripts/functions/misc/RewritePushdownRowsums.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/RewritePushdownRowsums.dml b/src/test/scripts/functions/misc/RewritePushdownRowsums.dml
new file mode 100644
index 0000000..945763e
--- /dev/null
+++ b/src/test/scripts/functions/misc/RewritePushdownRowsums.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+X = read($1);
+R = rowSums(t(X));
+
+write(R, $2);
\ No newline at end of file