You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/11/20 07:32:01 UTC
[2/2] systemml git commit: [SYSTEMML-2020] Fix codegen outer handling
of sparse side inputs
[SYSTEMML-2020] Fix codegen outer handling of sparse side inputs
The codegen outer template uses cache blocking for uncompressed input
data and column-wise access for compressed data. Both render the use of
side inputs with sparse cursors (row-major) invalid, leading to
incorrect results. This patch address this issue by using simple
dense/sparse side inputs only.
Furthermore, this patch also fixes the selection of the main driver for
codegen outer operations according to the involved operations. This
fixes issues with sparse side inputs that have less non-zeros than the
main driver input.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/9a6a9bd7
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/9a6a9bd7
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/9a6a9bd7
Branch: refs/heads/master
Commit: 9a6a9bd762820e55a845f78ff1e25278bdfc01c9
Parents: 079b493
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Nov 19 22:56:59 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Nov 19 22:56:59 2017 -0800
----------------------------------------------------------------------
.../codegen/template/TemplateOuterProduct.java | 10 +-
.../sysml/runtime/codegen/SpoofCellwise.java | 6 +
.../runtime/codegen/SpoofOuterProduct.java | 51 +++--
.../functions/codegen/SparseSideInputTest.java | 206 +++++++++++++++++++
.../functions/codegen/SparseSideInput1.R | 31 +++
.../functions/codegen/SparseSideInput1.dml | 27 +++
.../functions/codegen/SparseSideInput2.R | 31 +++
.../functions/codegen/SparseSideInput2.dml | 27 +++
.../functions/codegen/SparseSideInput3.R | 31 +++
.../functions/codegen/SparseSideInput3.dml | 27 +++
.../functions/codegen/SparseSideInput4.R | 33 +++
.../functions/codegen/SparseSideInput4.dml | 29 +++
.../functions/codegen/ZPackageSuite.java | 1 +
13 files changed, 487 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java
index 188bac2..12b8795 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java
@@ -178,10 +178,12 @@ public class TemplateOuterProduct extends TemplateBase {
CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID());
String primitiveOpName = ((BinaryOp)hop).getOp().toString();
- if( TemplateUtils.isMatrix(hop.getInput().get(0)) && cdata1 instanceof CNodeData )
- inHops2.put("_X", hop.getInput().get(0));
- if( TemplateUtils.isMatrix(hop.getInput().get(1)) && cdata2 instanceof CNodeData )
- inHops2.put("_X", hop.getInput().get(1));
+ if( HopRewriteUtils.isBinarySparseSafe(hop) ) {
+ if( TemplateUtils.isMatrix(hop.getInput().get(0)) && cdata1 instanceof CNodeData )
+ inHops2.put("_X", hop.getInput().get(0));
+ if( TemplateUtils.isMatrix(hop.getInput().get(1)) && cdata2 instanceof CNodeData )
+ inHops2.put("_X", hop.getInput().get(1));
+ }
//add lookups if required
cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0));
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
index 86c4ece..5acd5b4 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
@@ -375,6 +375,9 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl
MatrixBlock out, int m, int n, boolean sparseSafe, int rl, int ru)
throws DMLRuntimeException
{
+ //NOTE: we don't create sparse side inputs w/ row-major cursors because
+ //compressed data is access in a column-major order
+
if( _type == CellType.NO_AGG ) {
long lnnz = executeCompressedNoAgg(a, b, scalars, out, m, n, sparseSafe, rl, ru);
if( out.isInSparseFormat() )
@@ -401,6 +404,9 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl
private double executeCompressedAndAgg(CompressedMatrixBlock a, SideInput[] b, double[] scalars,
int m, int n, boolean sparseSafe, int rl, int ru) throws DMLRuntimeException
{
+ //NOTE: we don't create sparse side inputs w/ row-major cursors because
+ //compressed data is access in a column-major order
+
//numerically stable aggregation for sum/sum_sq
if( _aggOp == AggOp.SUM || _aggOp == AggOp.SUM_SQ )
return executeCompressedAggSum(a, b, scalars, m, n, sparseSafe, rl, ru);
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java
index 4d4da51..e99ebe1 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java
@@ -280,7 +280,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator
MatrixBlock a = inputs.get(0);
try
- {
+ {
ExecutorService pool = Executors.newFixedThreadPool(numThreads);
ArrayList<ParExecTask> tasks = new ArrayList<>();
//create tasks (for wdivmm-left, parallelization over columns;
@@ -345,7 +345,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator
//cache-conscious blocking: due to blocksize constraint (default 1000),
//a blocksize of 16 allows to fit blocks of UV into L2 cache (256KB)
- SideInput[] lb = createSparseSideInputs(b);
+ //NOTE: we don't create sparse side inputs w/ row-major cursors because
+ //cache blocking would lead to non-sequential access
+
final int blocksizeIJ = 16; //u/v block (max at typical L2 size)
int cix = 0;
//blocked execution
@@ -359,7 +361,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator
for( int j=bj, vix=bj*k; j<bjmin; j++, vix+=k)
if( a[ix+j] != 0 ) {
cix = (type == OutProdType.LEFT_OUTER_PRODUCT) ? vix : uix;
- genexecDense( a[ix+j], u, uix, v, vix, lb, scalars, c, cix, m, n, k, i, j);
+ genexecDense( a[ix+j], u, uix, v, vix, b, scalars, c, cix, m, n, k, i, j);
}
}
}
@@ -371,7 +373,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator
//cache-conscious blocking: due to blocksize constraint (default 1000),
//a blocksize of 16 allows to fit blocks of UV into L2 cache (256KB)
- SideInput[] lb = createSparseSideInputs(b);
+ //NOTE: we don't create sparse side inputs w/ row-major cursors because
+ //cache blocking would lead to non-sequential access
+
final int blocksizeIJ = 16; //u/v block (max at typical L2 size)
//blocked execution
double sum = 0;
@@ -385,9 +389,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator
for( int j=bj, vix=bj*k; j<bjmin; j++, vix+=k)
if( a[ix+j] != 0 ) {
if(type == OutProdType.CELLWISE_OUTER_PRODUCT)
- c[ix+j] = genexecCellwise( a[ix+j], u, uix, v, vix, lb, scalars, m, n, k, i, j );
+ c[ix+j] = genexecCellwise( a[ix+j], u, uix, v, vix, b, scalars, m, n, k, i, j );
else
- sum += genexecCellwise( a[ix+j], u, uix, v, vix, lb, scalars, m, n, k, i, j);
+ sum += genexecCellwise( a[ix+j], u, uix, v, vix, b, scalars, m, n, k, i, j);
}
}
if( type != OutProdType.CELLWISE_OUTER_PRODUCT )
@@ -397,7 +401,6 @@ public abstract class SpoofOuterProduct extends SpoofOperator
private void executeSparse(SparseBlock sblock, double[] u, double[] v, SideInput[] b, double[] scalars,
double[] c, int m, int n, int k, long nnz, OutProdType type, int rl, int ru, int cl, int cu)
{
- SideInput[] lb = createSparseSideInputs(b);
boolean left = (_outerProductType== OutProdType.LEFT_OUTER_PRODUCT);
//approach: iterate over non-zeros of w, selective mm computation
@@ -410,7 +413,8 @@ public abstract class SpoofOuterProduct extends SpoofOperator
{
//for ultra-sparse matrices, we do not allocate the index array because
//its allocation and maintenance can dominate the total runtime.
-
+ SideInput[] lb = createSparseSideInputs(b);
+
//core wdivmm block matrix mult
for( int i=rl, uix=rl*k; i<ru; i++, uix+=k ) {
if( sblock.isEmpty(i) ) continue;
@@ -430,6 +434,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator
}
else //sparse
{
+ //NOTE: we don't create sparse side inputs w/ row-major cursors because
+ //cache blocking would lead to non-sequential access
+
final int blocksizeJ = left ? Math.max(8,Math.min(L2_CACHESIZE/(k*8), blocksizeI)) : blocksizeI;
int[] curk = new int[Math.min(blocksizeI,ru-rl)];
@@ -457,7 +464,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator
int index = wpos + curk[i-bi];
for( ; index<wpos+wlen && wix[index]<bjmin; index++ ) {
- genexecDense(wval[index], u, uix, v, wix[index]*k, lb, scalars, c,
+ genexecDense(wval[index], u, uix, v, wix[index]*k, b, scalars, c,
(left ? wix[index]*k : uix), m, n, k, i, wix[index]);
}
curk[i-bi] = index - wpos;
@@ -470,7 +477,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator
private void executeCellwiseSparse(SparseBlock sblock, double[] u, double[] v, SideInput[] b, double[] scalars,
MatrixBlock out, int m, int n, int k, long nnz, OutProdType type, int rl, int ru, int cl, int cu )
{
- SideInput[] lb = createSparseSideInputs(b);
+ //NOTE: we don't create sparse side inputs w/ row-major cursors because
+ //cache blocking would lead to non-sequential access
+
final int blocksizeIJ = (int) (8L*m*n/nnz);
int[] curk = new int[Math.min(blocksizeIJ, ru-rl)];
@@ -495,11 +504,11 @@ public abstract class SpoofOuterProduct extends SpoofOperator
if( type == OutProdType.CELLWISE_OUTER_PRODUCT )
for( ; index<wpos+wlen && wix[index]<bjmin; index++ )
c[wix[index]] = genexecCellwise( wval[index],
- u, uix, v, wix[index]*k, lb, scalars, m, n, k, i, wix[index] );
+ u, uix, v, wix[index]*k, b, scalars, m, n, k, i, wix[index] );
else
for( ; index<wpos+wlen && wix[index]<bjmin; index++ )
tmp += genexecCellwise( wval[index],
- u, uix, v, wix[index]*k, lb, scalars, m, n, k, i, wix[index]);
+ u, uix, v, wix[index]*k, b, scalars, m, n, k, i, wix[index]);
curk[i-bi] = index - wpos;
}
}
@@ -526,7 +535,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator
int index = wpos + curk[i-bi];
for( ; index<wpos+wlen && wix[index]<bjmin; index++ ) {
c.append(i, wix[index], genexecCellwise( wval[index], u, uix, v,
- wix[index]*k, lb, scalars, m, n, k, i, wix[index] ));
+ wix[index]*k, b, scalars, m, n, k, i, wix[index] ));
}
curk[i-bi] = index - wpos;
}
@@ -538,7 +547,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator
private void executeCompressed(CompressedMatrixBlock a, double[] u, double[] v, SideInput[] b, double[] scalars,
double[] c, int m, int n, int k, OutProdType type, int rl, int ru, int cl, int cu)
{
- SideInput[] lb = createSparseSideInputs(b);
+ //NOTE: we don't create sparse side inputs w/ row-major cursors because
+ //compressed data is access in a column-major order
+
boolean left = (_outerProductType==OutProdType.LEFT_OUTER_PRODUCT);
Iterator<IJV> iter = !left ? a.getIterator(rl, ru, false) :
@@ -547,7 +558,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator
IJV cell = iter.next();
int uix = cell.getI() * k;
int vix = cell.getJ() * k;
- genexecDense(cell.getV(), u, uix, v, vix, lb, scalars, c,
+ genexecDense(cell.getV(), u, uix, v, vix, b, scalars, c,
left ? vix : uix, m, n, k, cell.getI(), cell.getJ());
}
}
@@ -555,7 +566,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator
private void executeCellwiseCompressed(CompressedMatrixBlock a, double[] u, double[] v, SideInput[] b, double[] scalars,
MatrixBlock out, int m, int n, int k, OutProdType type, int rl, int ru, int cl, int cu )
{
- SideInput[] lb = createSparseSideInputs(b);
+ //NOTE: we don't create sparse side inputs w/ row-major cursors because
+ //compressed data is access in a column-major order
+
double[] c = out.getDenseBlock();
SparseBlock csblock = out.getSparseBlock();
@@ -568,15 +581,15 @@ public abstract class SpoofOuterProduct extends SpoofOperator
if( out.isInSparseFormat() ) {
csblock.allocate(cell.getI());
csblock.append(cell.getI(), cell.getJ(),
- genexecCellwise(cell.getV(), u, uix, v, vix, lb, scalars, m, n, k, cell.getI(), cell.getJ()));
+ genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ()));
}
else {
c[cell.getI()*n+cell.getJ()] =
- genexecCellwise(cell.getV(), u, uix, v, vix, lb, scalars, m, n, k, cell.getI(), cell.getJ());
+ genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ());
}
}
else {
- c[0] += genexecCellwise(cell.getV(), u, uix, v, vix, lb, scalars, m, n, k, cell.getI(), cell.getJ());
+ c[0] += genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ());
}
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/java/org/apache/sysml/test/integration/functions/codegen/SparseSideInputTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/SparseSideInputTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/SparseSideInputTest.java
new file mode 100644
index 0000000..508cd52
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/SparseSideInputTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.codegen;
+
+import java.io.File;
+import java.util.HashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+public class SparseSideInputTest extends AutomatedTestBase
+{
+ private static final String TEST_NAME = "sparseSideInput";
+ private static final String TEST_NAME1 = TEST_NAME+"1"; //row sum(X/rowSums(X)+Y)
+ private static final String TEST_NAME2 = TEST_NAME+"2"; //cell sum(abs(X^2)+Y)
+ private static final String TEST_NAME3 = TEST_NAME+"3"; //magg sum(X^2), sum(X+Y)
+ private static final String TEST_NAME4 = TEST_NAME+"4"; //outer sum((X!=0) * (U %*% t(V) - Y))
+
+ private static final String TEST_DIR = "functions/codegen/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + SparseSideInputTest.class.getSimpleName() + "/";
+ private static String TEST_CONF1 = "SystemML-config-codegen.xml";
+ private static String TEST_CONF2 = "SystemML-config-codegen-compress.xml";
+ private static String TEST_CONF = TEST_CONF1;
+
+ private static final int rows = 1798;
+ private static final int cols = 784;
+ private static final double sparsity = 0.1;
+ private static final double eps = Math.pow(10, -7);
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ for(int i=1; i<=4; i++)
+ addTestConfiguration( TEST_NAME+i, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME+i, new String[] { String.valueOf(i) }) );
+ }
+
+ @Test
+ public void testCodegenRowULASparseSideInputCP() {
+ testCodegenIntegration( TEST_NAME1, false, ExecType.CP );
+ }
+
+ @Test
+ public void testCodegenRowCLASparseSideInputCP() {
+ testCodegenIntegration( TEST_NAME1, true, ExecType.CP );
+ }
+
+ @Test
+ public void testCodegenRowULASparseSideInputSP() {
+ testCodegenIntegration( TEST_NAME1, false, ExecType.SPARK );
+ }
+
+ @Test
+ public void testCodegenRowCLASparseSideInputSP() {
+ testCodegenIntegration( TEST_NAME1, true, ExecType.SPARK );
+ }
+
+ @Test
+ public void testCodegenCellULASparseSideInputCP() {
+ testCodegenIntegration( TEST_NAME2, false, ExecType.CP );
+ }
+
+ @Test
+ public void testCodegenCellCLASparseSideInputCP() {
+ testCodegenIntegration( TEST_NAME2, true, ExecType.CP );
+ }
+
+ @Test
+ public void testCodegenCellULASparseSideInputSP() {
+ testCodegenIntegration( TEST_NAME2, false, ExecType.SPARK );
+ }
+
+ @Test
+ public void testCodegenCellCLASparseSideInputSP() {
+ testCodegenIntegration( TEST_NAME2, true, ExecType.SPARK );
+ }
+
+ @Test
+ public void testCodegenMaggULASparseSideInputCP() {
+ testCodegenIntegration( TEST_NAME3, false, ExecType.CP );
+ }
+
+ @Test
+ public void testCodegenMaggCLASparseSideInputCP() {
+ testCodegenIntegration( TEST_NAME3, true, ExecType.CP );
+ }
+
+ @Test
+ public void testCodegenMaggULASparseSideInputSP() {
+ testCodegenIntegration( TEST_NAME3, false, ExecType.SPARK );
+ }
+
+ @Test
+ public void testCodegenMaggCLASparseSideInputSP() {
+ testCodegenIntegration( TEST_NAME3, true, ExecType.SPARK );
+ }
+
+ @Test
+ public void testCodegenOuterULASparseSideInputCP() {
+ testCodegenIntegration( TEST_NAME4, false, ExecType.CP );
+ }
+
+ @Test
+ public void testCodegenOuterCLASparseSideInputCP() {
+ testCodegenIntegration( TEST_NAME4, true, ExecType.CP );
+ }
+
+ @Test
+ public void testCodegenOuterULASparseSideInputSP() {
+ testCodegenIntegration( TEST_NAME4, false, ExecType.SPARK );
+ }
+
+ @Test
+ public void testCodegenOuterCLASparseSideInputSP() {
+ testCodegenIntegration( TEST_NAME4, true, ExecType.SPARK );
+ }
+
+ private void testCodegenIntegration( String testname, boolean compress, ExecType instType )
+ {
+ boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( instType ) {
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ TEST_CONF = compress ? TEST_CONF2 : TEST_CONF1;
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
+
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testname + ".dml";
+ programArgs = new String[]{"-explain", "-stats", "-args",
+ input("X"), input("Y"), output("R") };
+
+ fullRScriptName = HOME + testname + ".R";
+ rCmd = getRCmd(inputDir(), expectedDir());
+
+ //generate inputs
+ double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
+ double[][] Y = getRandomMatrix(rows, cols, 0, 1, sparsity, 3);
+ writeInputMatrixWithMTD("X", X, true);
+ writeInputMatrixWithMTD("Y", Y, true);
+
+ //run dml and r scripts
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare matrices
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
+ TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
+ Assert.assertTrue(heavyHittersContainsSubString("spoof")
+ || heavyHittersContainsSubString("sp_spoof"));
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
+ OptimizerUtils.ALLOW_AUTO_VECTORIZATION = true;
+ OptimizerUtils.ALLOW_OPERATOR_FUSION = true;
+ TEST_CONF = TEST_CONF2;
+ }
+ }
+
+ @Override
+ protected File getConfigTemplateFile() {
+ // Instrumentation in this test's output log to show custom configuration file used for template.
+ File f = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF);
+ System.out.println("This test case overrides default configuration with " + f.getPath());
+ return f;
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput1.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SparseSideInput1.R b/src/test/scripts/functions/codegen/SparseSideInput1.R
new file mode 100644
index 0000000..5acdb01
--- /dev/null
+++ b/src/test/scripts/functions/codegen/SparseSideInput1.R
@@ -0,0 +1,31 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X = readMM(paste(args[1], "X.mtx", sep=""))
+Y = readMM(paste(args[1], "Y.mtx", sep=""))
+
+R = as.matrix(sum(X/rowSums(X)+Y));
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""), format="text")
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput1.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SparseSideInput1.dml b/src/test/scripts/functions/codegen/SparseSideInput1.dml
new file mode 100644
index 0000000..1e0b241
--- /dev/null
+++ b/src/test/scripts/functions/codegen/SparseSideInput1.dml
@@ -0,0 +1,27 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+Y = read($2);
+
+R = as.matrix(sum(X/rowSums(X)+Y));
+
+write(R, $3)
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput2.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SparseSideInput2.R b/src/test/scripts/functions/codegen/SparseSideInput2.R
new file mode 100644
index 0000000..55872a4
--- /dev/null
+++ b/src/test/scripts/functions/codegen/SparseSideInput2.R
@@ -0,0 +1,31 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X = readMM(paste(args[1], "X.mtx", sep=""))
+Y = readMM(paste(args[1], "Y.mtx", sep=""))
+
+R = as.matrix(sum(abs(X^2)+Y));
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""), format="text")
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SparseSideInput2.dml b/src/test/scripts/functions/codegen/SparseSideInput2.dml
new file mode 100644
index 0000000..9c0ec59
--- /dev/null
+++ b/src/test/scripts/functions/codegen/SparseSideInput2.dml
@@ -0,0 +1,27 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+Y = read($2);
+
+R = as.matrix(sum(abs(X^2)+Y));
+
+write(R, $3)
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput3.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SparseSideInput3.R b/src/test/scripts/functions/codegen/SparseSideInput3.R
new file mode 100644
index 0000000..106e59c
--- /dev/null
+++ b/src/test/scripts/functions/codegen/SparseSideInput3.R
@@ -0,0 +1,31 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X = readMM(paste(args[1], "X.mtx", sep=""))
+Y = readMM(paste(args[1], "Y.mtx", sep=""))
+
+R = as.matrix(sum(X^2)+sum(X+Y));
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""), format="text")
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput3.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SparseSideInput3.dml b/src/test/scripts/functions/codegen/SparseSideInput3.dml
new file mode 100644
index 0000000..253ba49
--- /dev/null
+++ b/src/test/scripts/functions/codegen/SparseSideInput3.dml
@@ -0,0 +1,27 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+Y = read($2);
+
+R = as.matrix(sum(X^2)+sum(X+Y));
+
+write(R, $3)
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput4.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SparseSideInput4.R b/src/test/scripts/functions/codegen/SparseSideInput4.R
new file mode 100644
index 0000000..a109983
--- /dev/null
+++ b/src/test/scripts/functions/codegen/SparseSideInput4.R
@@ -0,0 +1,33 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X = readMM(paste(args[1], "X.mtx", sep=""))
+Y = readMM(paste(args[1], "Y.mtx", sep=""))
+
+U = matrix(1, nrow(X), 10);
+V = matrix(1, ncol(X), 10);
+R = as.matrix(sum((X!=0)*((U%*%t(V))/10+Y)));
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""), format="text")
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput4.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SparseSideInput4.dml b/src/test/scripts/functions/codegen/SparseSideInput4.dml
new file mode 100644
index 0000000..37c25ce
--- /dev/null
+++ b/src/test/scripts/functions/codegen/SparseSideInput4.dml
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+Y = read($2);
+
+U = matrix(1, nrow(X), 10);
+V = matrix(1, ncol(X), 10);
+R = as.matrix(sum((X!=0)*((U%*%t(V))/10+Y)));
+
+write(R, $3)
http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java
index 75b66a1..596d673 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java
@@ -51,6 +51,7 @@ import org.junit.runners.Suite;
OuterProdTmplTest.class,
RowAggTmplTest.class,
RowVectorComparisonTest.class,
+ SparseSideInputTest.class,
SumProductChainTest.class,
})