You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/04/14 19:45:02 UTC
[1/3] incubator-systemml git commit: [SYSTEMML-1515] Generalized
codegen cell template (sideways row vectors)
Repository: incubator-systemml
Updated Branches:
refs/heads/master 5db91308f -> 149562eca
[SYSTEMML-1515] Generalized codegen cell template (sideways row vectors)
This patch generalizes the existing codegen cell template by allowing
sideways row vectors, i.e., matrix-row vector binary operations, in
addition to sideways column vectors and matrices.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/b70ee453
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/b70ee453
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/b70ee453
Branch: refs/heads/master
Commit: b70ee45330a457e3f0ee61c499306fb2518997c7
Parents: 5db9130
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Apr 13 14:18:47 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Apr 14 12:46:36 2017 -0700
----------------------------------------------------------------------
.../sysml/hops/codegen/SpoofCompiler.java | 6 +--
.../sysml/hops/codegen/cplan/CNodeUnary.java | 6 ++-
.../hops/codegen/template/TemplateCell.java | 42 +++++---------------
.../hops/codegen/template/TemplateRow.java | 24 +++--------
.../hops/codegen/template/TemplateUtils.java | 20 ++++++++--
.../functions/codegen/CellwiseTmplTest.java | 24 ++++++++++-
.../scripts/functions/codegen/cellwisetmpl14.R | 31 +++++++++++++++
.../functions/codegen/cellwisetmpl14.dml | 27 +++++++++++++
8 files changed, 118 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b70ee453/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java b/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
index fdb8d9d..1f0644b 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
@@ -41,8 +41,6 @@ import org.apache.sysml.hops.codegen.cplan.CNodeOuterProduct;
import org.apache.sysml.hops.codegen.cplan.CNodeTernary;
import org.apache.sysml.hops.codegen.cplan.CNodeTernary.TernaryType;
import org.apache.sysml.hops.codegen.cplan.CNodeTpl;
-import org.apache.sysml.hops.codegen.cplan.CNodeUnary;
-import org.apache.sysml.hops.codegen.cplan.CNodeUnary.UnaryType;
import org.apache.sysml.hops.codegen.template.TemplateBase;
import org.apache.sysml.hops.codegen.template.TemplateBase.CloseType;
import org.apache.sysml.hops.codegen.template.TemplateBase.TemplateType;
@@ -641,9 +639,7 @@ public class SpoofCompiler
private static void rFindAndRemoveLookup(CNode node, CNodeData mainInput) {
for( int i=0; i<node.getInput().size(); i++ ) {
CNode tmp = node.getInput().get(i);
- if( tmp instanceof CNodeUnary && (((CNodeUnary)tmp).getType()==UnaryType.LOOKUP_R
- || ((CNodeUnary)tmp).getType()==UnaryType.LOOKUP_RC)
- && tmp.getInput().get(0) instanceof CNodeData
+ if( TemplateUtils.isLookup(tmp) && tmp.getInput().get(0) instanceof CNodeData
&& ((CNodeData)tmp.getInput().get(0)).getHopID()==mainInput.getHopID() )
{
node.getInput().set(i, tmp.getInput().get(0));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b70ee453/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
index 30752a2..7808421 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
@@ -28,7 +28,7 @@ import org.apache.sysml.parser.Expression.DataType;
public class CNodeUnary extends CNode
{
public enum UnaryType {
- LOOKUP_R, LOOKUP_RC, LOOKUP0, //codegen specific
+ LOOKUP_R, LOOKUP_C, LOOKUP_RC, LOOKUP0, //codegen specific
ROW_SUMS, ROW_MINS, ROW_MAXS, //codegen specific
VECT_EXP, VECT_POW2, VECT_MULT2, VECT_SQRT, VECT_LOG,
VECT_ABS, VECT_ROUND, VECT_CEIL, VECT_FLOOR, VECT_SIGN,
@@ -73,6 +73,8 @@ public class CNodeUnary extends CNode
return " double %TMP% = FastMath.exp(%IN1%);\n";
case LOOKUP_R:
return " double %TMP% = getValue(%IN1%, rowIndex);\n";
+ case LOOKUP_C:
+ return " double %TMP% = getValue(%IN1%, colIndex);\n";
case LOOKUP_RC:
return " double %TMP% = getValue(%IN1%, rowIndex*n+colIndex);\n";
case LOOKUP0:
@@ -207,6 +209,7 @@ public class CNodeUnary extends CNode
case VECT_FLOOR:
case VECT_SIGN: return "u(v"+_type.name().toLowerCase()+")";
case LOOKUP_R: return "u(ixr)";
+ case LOOKUP_C: return "u(ixc)";
case LOOKUP_RC: return "u(ixrc)";
case LOOKUP0: return "u(ix0)";
case POW2: return "^2";
@@ -237,6 +240,7 @@ public class CNodeUnary extends CNode
case ROW_MAXS:
case EXP:
case LOOKUP_R:
+ case LOOKUP_C:
case LOOKUP_RC:
case LOOKUP0:
case POW2:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b70ee453/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
index 95f6643..d5ac99c 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
@@ -165,10 +165,7 @@ public class TemplateCell extends TemplateBase
if(hop instanceof UnaryOp)
{
CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID());
- if( TemplateUtils.isColVector(cdata1) )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
- else if( cdata1 instanceof CNodeData && hop.getInput().get(0).getDataType().isMatrix() )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_RC);
+ cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
String primitiveOpName = ((UnaryOp)hop).getOp().name();
out = new CNodeUnary(cdata1, UnaryType.valueOf(primitiveOpName));
@@ -180,17 +177,9 @@ public class TemplateCell extends TemplateBase
CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
String primitiveOpName = bop.getOp().name();
- //cdata1 is vector
- if( TemplateUtils.isColVector(cdata1) )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
- else if( cdata1 instanceof CNodeData && hop.getInput().get(0).getDataType().isMatrix() )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_RC);
-
- //cdata2 is vector
- if( TemplateUtils.isColVector(cdata2) )
- cdata2 = new CNodeUnary(cdata2, UnaryType.LOOKUP_R);
- else if( cdata2 instanceof CNodeData && hop.getInput().get(1).getDataType().isMatrix() )
- cdata2 = new CNodeUnary(cdata2, UnaryType.LOOKUP_RC);
+ //add lookups if required
+ cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
+ cdata2 = TemplateUtils.wrapLookupIfNecessary(cdata2, hop.getInput().get(1));
if( bop.getOp()==OpOp2.POW && cdata2.isLiteral() && cdata2.getVarname().equals("2") )
out = new CNodeUnary(cdata1, UnaryType.POW2);
@@ -206,17 +195,9 @@ public class TemplateCell extends TemplateBase
CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
CNode cdata3 = tmp.get(hop.getInput().get(2).getHopID());
- //cdata1 is vector
- if( TemplateUtils.isColVector(cdata1) )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
- else if( cdata1 instanceof CNodeData && hop.getInput().get(0).getDataType().isMatrix() )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_RC);
-
- //cdata3 is vector
- if( TemplateUtils.isColVector(cdata3) )
- cdata3 = new CNodeUnary(cdata3, UnaryType.LOOKUP_R);
- else if( cdata3 instanceof CNodeData && hop.getInput().get(2).getDataType().isMatrix() )
- cdata3 = new CNodeUnary(cdata3, UnaryType.LOOKUP_RC);
+ //add lookups if required
+ cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
+ cdata3 = TemplateUtils.wrapLookupIfNecessary(cdata3, hop.getInput().get(2));
//construct ternary cnode, primitive operation derived from OpOp3
out = new CNodeTernary(cdata1, cdata2, cdata3,
@@ -225,10 +206,7 @@ public class TemplateCell extends TemplateBase
else if( hop instanceof ParameterizedBuiltinOp )
{
CNode cdata1 = tmp.get(((ParameterizedBuiltinOp)hop).getTargetHop().getHopID());
- if( TemplateUtils.isColVector(cdata1) )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
- else if( cdata1 instanceof CNodeData && hop.getInput().get(0).getDataType().isMatrix() )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_RC);
+ cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
CNode cdata2 = tmp.get(((ParameterizedBuiltinOp)hop).getParameterHop("pattern").getHopID());
CNode cdata3 = tmp.get(((ParameterizedBuiltinOp)hop).getParameterHop("replacement").getHopID());
@@ -290,8 +268,8 @@ public class TemplateCell extends TemplateBase
isBinaryMatrixScalar = (ldt.isScalar() || rdt.isScalar());
isBinaryMatrixVector = hop.dimsKnown()
- && ((ldt.isMatrix() && TemplateUtils.isVectorOrScalar(right) && !TemplateUtils.isBinaryMatrixRowVector(hop))
- || (rdt.isMatrix() && TemplateUtils.isVectorOrScalar(left) && !TemplateUtils.isBinaryMatrixRowVector(hop)) );
+ && ((ldt.isMatrix() && TemplateUtils.isVectorOrScalar(right))
+ || (rdt.isMatrix() && TemplateUtils.isVectorOrScalar(left)) );
isBinaryMatrixMatrixDense = hop.dimsKnown() && HopRewriteUtils.isEqualSize(left, right)
&& ldt.isMatrix() && rdt.isMatrix() && !HopRewriteUtils.isSparse(left) && !HopRewriteUtils.isSparse(right);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b70ee453/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java
index ca9776d..3af8be4 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java
@@ -227,10 +227,7 @@ public class TemplateRow extends TemplateBase
}
else //general scalar case
{
- if( TemplateUtils.isColVector(cdata1) )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
- else if( cdata1 instanceof CNodeData && hop.getInput().get(0).getDataType().isMatrix() )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_RC);
+ cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
String primitiveOpName = ((UnaryOp)hop).getOp().toString();
out = new CNodeUnary(cdata1, UnaryType.valueOf(primitiveOpName));
@@ -271,17 +268,9 @@ public class TemplateRow extends TemplateBase
CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
CNode cdata3 = tmp.get(hop.getInput().get(2).getHopID());
- //cdata1 is vector
- if( TemplateUtils.isColVector(cdata1) )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
- else if( cdata1 instanceof CNodeData && hop.getInput().get(0).getDataType().isMatrix() )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_RC);
-
- //cdata3 is vector
- if( TemplateUtils.isColVector(cdata3) )
- cdata3 = new CNodeUnary(cdata3, UnaryType.LOOKUP_R);
- else if( cdata3 instanceof CNodeData && hop.getInput().get(2).getDataType().isMatrix() )
- cdata3 = new CNodeUnary(cdata3, UnaryType.LOOKUP_RC);
+ //add lookups if required
+ cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
+ cdata3 = TemplateUtils.wrapLookupIfNecessary(cdata3, hop.getInput().get(2));
//construct ternary cnode, primitive operation derived from OpOp3
out = new CNodeTernary(cdata1, cdata2, cdata3,
@@ -290,10 +279,7 @@ public class TemplateRow extends TemplateBase
else if( hop instanceof ParameterizedBuiltinOp )
{
CNode cdata1 = tmp.get(((ParameterizedBuiltinOp)hop).getTargetHop().getHopID());
- if( TemplateUtils.isColVector(cdata1) )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R);
- else if( cdata1 instanceof CNodeData && hop.getInput().get(0).getDataType().isMatrix() )
- cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_RC);
+ cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
CNode cdata2 = tmp.get(((ParameterizedBuiltinOp)hop).getParameterHop("pattern").getHopID());
CNode cdata3 = tmp.get(((ParameterizedBuiltinOp)hop).getParameterHop("replacement").getHopID());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b70ee453/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java
index 8811cb8..0a19b56 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java
@@ -74,6 +74,17 @@ public class TemplateUtils
&& hop.getNumRows() == 1 && hop.getNumCols() != 1);
}
+ public static CNode wrapLookupIfNecessary(CNode node, Hop hop) {
+ CNode ret = node;
+ if( isColVector(node) )
+ ret = new CNodeUnary(node, UnaryType.LOOKUP_R);
+ else if( isRowVector(node) )
+ ret = new CNodeUnary(node, UnaryType.LOOKUP_C);
+ else if( node instanceof CNodeData && hop.getDataType().isMatrix() )
+ ret = new CNodeUnary(node, UnaryType.LOOKUP_RC);
+ return ret;
+ }
+
public static boolean isMatrix(Hop hop) {
return (hop.getDataType() == DataType.MATRIX && hop.getDim1() != 1 && hop.getDim2()!=1);
}
@@ -256,9 +267,12 @@ public class TemplateUtils
}
public static boolean isLookup(CNode node) {
- return (node instanceof CNodeUnary
- && (((CNodeUnary)node).getType()==UnaryType.LOOKUP_R
- || ((CNodeUnary)node).getType()==UnaryType.LOOKUP_RC));
+ return isUnary(node, UnaryType.LOOKUP_R, UnaryType.LOOKUP_C, UnaryType.LOOKUP_RC);
+ }
+
+ public static boolean isUnary(CNode node, UnaryType...types) {
+ return node instanceof CNodeUnary
+ && ArrayUtils.contains(types, ((CNodeUnary)node).getType());
}
public static CNodeData createCNodeData(Hop hop, boolean compileLiterals) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b70ee453/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
index fc41837..fbd456f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
@@ -49,7 +49,7 @@ public class CellwiseTmplTest extends AutomatedTestBase
private static final String TEST_NAME11 = TEST_NAME+11; //replace((0 / (X - 500))+1, 0/0, 7)
private static final String TEST_NAME12 = TEST_NAME+12; //((X/3) %% 0.6) + ((X/3) %/% 0.6)
private static final String TEST_NAME13 = TEST_NAME+13; //min(X + 7 * Y) large
-
+ private static final String TEST_NAME14 = TEST_NAME+14; //-2 * X + t(Y); t(Y) is rowvector
private static final String TEST_DIR = "functions/codegen/";
private static final String TEST_CLASS_DIR = TEST_DIR + CellwiseTmplTest.class.getSimpleName() + "/";
@@ -62,7 +62,7 @@ public class CellwiseTmplTest extends AutomatedTestBase
@Override
public void setUp() {
TestUtils.clearAssertionInformation();
- for( int i=1; i<=13; i++ ) {
+ for( int i=1; i<=14; i++ ) {
addTestConfiguration( TEST_NAME+i, new TestConfiguration(
TEST_CLASS_DIR, TEST_NAME+i, new String[] {String.valueOf(i)}) );
}
@@ -133,6 +133,11 @@ public class CellwiseTmplTest extends AutomatedTestBase
public void testCodegenCellwiseRewrite13() {
testCodegenIntegration( TEST_NAME13, true, ExecType.CP );
}
+
+ @Test
+ public void testCodegenCellwiseRewrite14() {
+ testCodegenIntegration( TEST_NAME14, true, ExecType.CP );
+ }
@Test
public void testCodegenCellwise1() {
@@ -199,6 +204,11 @@ public class CellwiseTmplTest extends AutomatedTestBase
public void testCodegenCellwise13() {
testCodegenIntegration( TEST_NAME13, false, ExecType.CP );
}
+
+ @Test
+ public void testCodegenCellwise14() {
+ testCodegenIntegration( TEST_NAME14, false, ExecType.CP );
+ }
@Test
public void testCodegenCellwiseRewrite1_sp() {
@@ -235,6 +245,16 @@ public class CellwiseTmplTest extends AutomatedTestBase
testCodegenIntegration( TEST_NAME12, true, ExecType.SPARK );
}
+ @Test
+ public void testCodegenCellwiseRewrite13_sp() {
+ testCodegenIntegration( TEST_NAME13, true, ExecType.SPARK );
+ }
+
+ @Test
+ public void testCodegenCellwiseRewrite14_sp() {
+ testCodegenIntegration( TEST_NAME14, true, ExecType.SPARK );
+ }
+
private void testCodegenIntegration( String testname, boolean rewrites, ExecType instType )
{
boolean oldRewrites = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b70ee453/src/test/scripts/functions/codegen/cellwisetmpl14.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/cellwisetmpl14.R b/src/test/scripts/functions/codegen/cellwisetmpl14.R
new file mode 100644
index 0000000..d649da8
--- /dev/null
+++ b/src/test/scripts/functions/codegen/cellwisetmpl14.R
@@ -0,0 +1,31 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args<-commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X = matrix(seq(7, 2200006), 1100, 2000, byrow=TRUE);
+Y = seq(1, 2000);
+
+R = -2 * X + (matrix(1,nrow(X),1) %*% t(Y));
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "S", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b70ee453/src/test/scripts/functions/codegen/cellwisetmpl14.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/cellwisetmpl14.dml b/src/test/scripts/functions/codegen/cellwisetmpl14.dml
new file mode 100644
index 0000000..1c0597b
--- /dev/null
+++ b/src/test/scripts/functions/codegen/cellwisetmpl14.dml
@@ -0,0 +1,27 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = matrix(seq(7, 2200006), 1100, 2000);
+Y = seq(1, 2000);
+
+R = -2 * X + t(Y);
+
+write(R, $1)
[2/3] incubator-systemml git commit: [SYSTEMML-1518] Fix corrupted
input file names in old/new mlcontext
Posted by mb...@apache.org.
[SYSTEMML-1518] Fix corrupted input file names in old/new mlcontext
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/55e46e7c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/55e46e7c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/55e46e7c
Branch: refs/heads/master
Commit: 55e46e7cbeb46073d5d2f19fc4c3b59bdc01c7cb
Parents: b70ee45
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Apr 14 01:04:22 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Apr 14 12:46:39 2017 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/api/MLContext.java | 6 +++---
src/main/java/org/apache/sysml/hops/OptimizerUtils.java | 6 +++++-
src/main/java/org/apache/sysml/lops/compile/Dag.java | 6 +++++-
3 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/55e46e7c/src/main/java/org/apache/sysml/api/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java
index 85827da..809776a 100644
--- a/src/main/java/org/apache/sysml/api/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/MLContext.java
@@ -518,7 +518,7 @@ public class MLContext {
if( format.equals("csv") ) {
int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, blksz, blksz, nnz);
- mo = new MatrixObject(ValueType.DOUBLE, null, new MatrixFormatMetaData(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
+ mo = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), new MatrixFormatMetaData(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
}
else if( format.equals("text") ) {
if(rlen == -1 || clen == -1) {
@@ -526,7 +526,7 @@ public class MLContext {
}
int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, blksz, blksz, nnz);
- mo = new MatrixObject(ValueType.DOUBLE, null, new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
+ mo = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
}
else if( format.equals("mm") ) {
// TODO: Handle matrix market
@@ -588,7 +588,7 @@ public class MLContext {
if(rlen == -1 || clen == -1) {
throw new DMLRuntimeException("The metadata is required in registerInput for format:" + format);
}
- fo = new FrameObject(null, new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
+ fo = new FrameObject(OptimizerUtils.getUniqueTempFileName(), new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
}
else {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/55e46e7c/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index 1a959b6..3691897 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -41,6 +41,7 @@ import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.functionobjects.IntegerDivide;
import org.apache.sysml.runtime.functionobjects.Modulus;
@@ -885,7 +886,10 @@ public class OptimizerUtils
* @return unique temp file name
*/
public static String getUniqueTempFileName() {
- return new Dag<Lop>().getNextUniqueFilename();
+ return ConfigurationManager.getScratchSpace()
+ + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID()
+ + Lop.FILE_SEPARATOR + ProgramConverter.CP_ROOT_THREAD_ID + Lop.FILE_SEPARATOR
+ + Dag.getNextUniqueFilenameSuffix();
}
public static boolean allowsToFilterEmptyBlockOutputs( Hop hop )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/55e46e7c/src/main/java/org/apache/sysml/lops/compile/Dag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/Dag.java b/src/main/java/org/apache/sysml/lops/compile/Dag.java
index 71171d0..5816f3f 100644
--- a/src/main/java/org/apache/sysml/lops/compile/Dag.java
+++ b/src/main/java/org/apache/sysml/lops/compile/Dag.java
@@ -205,8 +205,12 @@ public class Dag<N extends Lop>
return scratchFilePath;
}
+ public static String getNextUniqueFilenameSuffix() {
+ return "temp" + job_id.getNextID();
+ }
+
public String getNextUniqueFilename() {
- return getFilePath() + "temp" + job_id.getNextID();
+ return getFilePath() + getNextUniqueFilenameSuffix();
}
public static String getNextUniqueVarname(DataType dt) {
[3/3] incubator-systemml git commit: [SYSTEMML-1520] Fix sparse
output corruption of row-wise rexpand
Posted by mb...@apache.org.
[SYSTEMML-1520] Fix sparse output corruption of row-wise rexpand
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/149562ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/149562ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/149562ec
Branch: refs/heads/master
Commit: 149562eca0bb1a438e34dc6e2d7993b44e2e612e
Parents: 55e46e7
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Apr 14 02:24:13 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Apr 14 12:46:42 2017 -0700
----------------------------------------------------------------------
.../org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/149562ec/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
index 9c6af97..dfd19aa 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
@@ -1899,9 +1899,13 @@ public class LibMatrixReorg
if( val == Math.floor(val) && val >= 1 && val <= max )
ret.appendValue((int)(val-1), i+tmpi[j], 1);
}
-
}
+ //ensure valid output sparse representation
+ //(necessary due to cache-conscious processing w/ unstable sort)
+ if( ret.isInSparseFormat() )
+ ret.sortSparseRows();
+
return ret;
}
@@ -1940,7 +1944,7 @@ public class LibMatrixReorg
{
//copy value array from input matrix
if( in.isEmptyBlock(false) ) {
- Arrays.fill(tmp, 0, 0, len);
+ Arrays.fill(tmp, 0, len, 0);
}
else if( in.sparse ){ //SPARSE
for( int i=0; i<len; i++ )