You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/04/07 04:15:14 UTC
[1/3] incubator-systemml git commit: [SYSTEMML-1473] Fix correctness
codegen cellwise min/max aggregation
Repository: incubator-systemml
Updated Branches:
refs/heads/master 5744e752d -> 9820f4c52
[SYSTEMML-1473] Fix correctness codegen cellwise min/max aggregation
The cellwise codegen template allows for sum, sum_sq, min and max as
aggregation functions. In case of multi-threaded execution there is a
need for a final aggregation of partial results. This patch fixes
resulting correctness issues for min and max in such scenarios.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7c153390
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7c153390
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7c153390
Branch: refs/heads/master
Commit: 7c15339019360f3add4aac23b06cde45a4b758f8
Parents: f2ea633
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Apr 6 18:51:45 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Apr 6 21:16:30 2017 -0700
----------------------------------------------------------------------
.../sysml/runtime/codegen/SpoofCellwise.java | 25 +++++++++------
.../functions/codegen/CellwiseTmplTest.java | 16 ++++++++--
.../scripts/functions/codegen/cellwisetmpl13.R | 33 ++++++++++++++++++++
.../functions/codegen/cellwisetmpl13.dml | 29 +++++++++++++++++
4 files changed, 92 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7c153390/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
index 67dd346..12d14ad 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
@@ -122,10 +122,10 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl
boolean sparseSafe = isSparseSafe() || (b.length == 0
&& genexec( 0, b, scalars, m, n, 0, 0 ) == 0);
- double sum = 0;
+ double ret = 0;
if( k <= 1 ) //SINGLE-THREADED
{
- sum = ( !inputs.get(0).isInSparseFormat() ) ?
+ ret = ( !inputs.get(0).isInSparseFormat() ) ?
executeDenseAndAgg(inputs.get(0).getDenseBlock(), b, scalars, m, n, sparseSafe, 0, m) :
executeSparseAndAgg(inputs.get(0).getSparseBlock(), b, scalars, m, n, sparseSafe, 0, m);
}
@@ -143,11 +143,18 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl
pool.shutdown();
//aggregate partial results
- KahanObject kbuff = new KahanObject(0, 0);
- KahanPlus kplus = KahanPlus.getKahanPlusFnObject();
- for( Future<Double> task : taskret )
- kplus.execute2(kbuff, task.get());
- sum = kbuff._sum;
+ ValueFunction vfun = getAggFunction();
+ if( vfun instanceof KahanFunction ) {
+ KahanObject kbuff = new KahanObject(0, 0);
+ KahanPlus kplus = KahanPlus.getKahanPlusFnObject();
+ for( Future<Double> task : taskret )
+ kplus.execute2(kbuff, task.get());
+ ret = kbuff._sum;
+ }
+ else {
+ for( Future<Double> task : taskret )
+ ret = vfun.execute(ret, task.get());
+ }
}
catch(Exception ex) {
throw new DMLRuntimeException(ex);
@@ -157,9 +164,9 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl
//correction for min/max
if( (_aggOp == AggOp.MIN || _aggOp == AggOp.MAX) && sparseSafe
&& inputs.get(0).getNonZeros()<inputs.get(0).getNumRows()*inputs.get(0).getNumColumns() )
- sum = getAggFunction().execute(sum, 0); //unseen 0 might be max or min value
+ ret = getAggFunction().execute(ret, 0); //unseen 0 might be max or min value
- return new DoubleObject(sum);
+ return new DoubleObject(ret);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7c153390/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
index 6468003..fc41837 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
@@ -48,7 +48,9 @@ public class CellwiseTmplTest extends AutomatedTestBase
private static final String TEST_NAME10 = TEST_NAME+10; //min/max(X + 7 * Y)
private static final String TEST_NAME11 = TEST_NAME+11; //replace((0 / (X - 500))+1, 0/0, 7)
private static final String TEST_NAME12 = TEST_NAME+12; //((X/3) %% 0.6) + ((X/3) %/% 0.6)
-
+ private static final String TEST_NAME13 = TEST_NAME+13; //min(X + 7 * Y) large
+
+
private static final String TEST_DIR = "functions/codegen/";
private static final String TEST_CLASS_DIR = TEST_DIR + CellwiseTmplTest.class.getSimpleName() + "/";
private final static String TEST_CONF6 = "SystemML-config-codegen6.xml";
@@ -60,7 +62,7 @@ public class CellwiseTmplTest extends AutomatedTestBase
@Override
public void setUp() {
TestUtils.clearAssertionInformation();
- for( int i=1; i<=12; i++ ) {
+ for( int i=1; i<=13; i++ ) {
addTestConfiguration( TEST_NAME+i, new TestConfiguration(
TEST_CLASS_DIR, TEST_NAME+i, new String[] {String.valueOf(i)}) );
}
@@ -126,6 +128,11 @@ public class CellwiseTmplTest extends AutomatedTestBase
public void testCodegenCellwiseRewrite12() {
testCodegenIntegration( TEST_NAME12, true, ExecType.CP );
}
+
+ @Test
+ public void testCodegenCellwiseRewrite13() {
+ testCodegenIntegration( TEST_NAME13, true, ExecType.CP );
+ }
@Test
public void testCodegenCellwise1() {
@@ -187,6 +194,11 @@ public class CellwiseTmplTest extends AutomatedTestBase
public void testCodegenCellwise12() {
testCodegenIntegration( TEST_NAME12, false, ExecType.CP );
}
+
+ @Test
+ public void testCodegenCellwise13() {
+ testCodegenIntegration( TEST_NAME13, false, ExecType.CP );
+ }
@Test
public void testCodegenCellwiseRewrite1_sp() {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7c153390/src/test/scripts/functions/codegen/cellwisetmpl13.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/cellwisetmpl13.R b/src/test/scripts/functions/codegen/cellwisetmpl13.R
new file mode 100644
index 0000000..527bb13
--- /dev/null
+++ b/src/test/scripts/functions/codegen/cellwisetmpl13.R
@@ -0,0 +1,33 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args<-commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X = matrix(seq(7, 1100006), 1100, 1000, byrow=TRUE);
+Y = matrix(seq(6, 1100005), 1100, 1000, byrow=TRUE);
+
+Z = X + -7 * Y;
+R1 = min(Z);
+R = as.matrix(R1);
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "S", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7c153390/src/test/scripts/functions/codegen/cellwisetmpl13.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/cellwisetmpl13.dml b/src/test/scripts/functions/codegen/cellwisetmpl13.dml
new file mode 100644
index 0000000..58037dd
--- /dev/null
+++ b/src/test/scripts/functions/codegen/cellwisetmpl13.dml
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = matrix(seq(7, 1100006), 1100, 1000);
+Y = matrix(seq(6, 1100005), 1100, 1000);
+
+Z = X + -7 * Y;
+R1 = min(Z);
+R = as.matrix(R1);
+
+write(R, $1)
[2/3] incubator-systemml git commit: [SYSTEMML-1472] Improved codegen
handling of empty inputs w/o alloc
Posted by mb...@apache.org.
[SYSTEMML-1472] Improved codegen handling of empty inputs w/o alloc
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f2ea6336
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f2ea6336
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f2ea6336
Branch: refs/heads/master
Commit: f2ea6336fea757ba0fb81f315af9bb97df6615d3
Parents: 5744e75
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Apr 6 17:14:45 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Apr 6 21:16:30 2017 -0700
----------------------------------------------------------------------
.../org/apache/sysml/hops/codegen/cplan/CNodeTernary.java | 2 +-
.../org/apache/sysml/hops/codegen/cplan/CNodeUnary.java | 4 ++--
.../org/apache/sysml/runtime/codegen/SpoofOperator.java | 9 +++++++--
.../integration/functions/codegen/MultiAggTmplTest.java | 8 ++++----
4 files changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f2ea6336/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java
index a8bbcb2..b3b1942 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java
@@ -54,7 +54,7 @@ public class CNodeTernary extends CNode
return " double %TMP% = Double.isNaN(%IN1%) ? %IN3% : %IN1%;\n";
case LOOKUP_RC1:
- return " double %TMP% = %IN1%[rowIndex*%IN2%+%IN3%-1];\n";
+ return " double %TMP% = getValue(%IN1%, rowIndex*%IN2%+%IN3%-1);\n";
default:
throw new RuntimeException("Invalid ternary type: "+this.toString());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f2ea6336/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
index e28a989..9d3a877 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
@@ -48,9 +48,9 @@ public class CNodeUnary extends CNode
case EXP:
return " double %TMP% = FastMath.exp(%IN1%);\n";
case LOOKUP_R:
- return " double %TMP% = %IN1%[rowIndex];\n";
+ return " double %TMP% = getValue(%IN1%, rowIndex);\n";
case LOOKUP_RC:
- return " double %TMP% = %IN1%[rowIndex*n+colIndex];\n";
+ return " double %TMP% = getValue(%IN1%, rowIndex*n+colIndex);\n";
case LOOKUP0:
return " double %TMP% = %IN1%[0];\n" ;
case POW2:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f2ea6336/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java
index c62a535..ced10f9 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java
@@ -70,8 +70,7 @@ public abstract class SpoofOperator implements Serializable
for(int i=offset; i<offset+len; i++) {
//convert empty or sparse to dense temporary block (note: we don't do
//this in place because this block might be used by multiple threads)
- if( (inputs.get(i).isEmptyBlock(false) && !inputs.get(i).isAllocated())
- || inputs.get(i).isInSparseFormat() ) {
+ if( inputs.get(i).isInSparseFormat() && inputs.get(i).isAllocated() ) {
MatrixBlock tmp = inputs.get(i);
b[i-offset] = DataConverter.convertToDoubleVector(tmp);
LOG.warn(getClass().getName()+": Converted "+tmp.getNumRows()+"x"+tmp.getNumColumns()+
@@ -92,4 +91,10 @@ public abstract class SpoofOperator implements Serializable
scalars[i] = scalarObjects.get(i).getDoubleValue();
return scalars;
}
+
+ //abstraction for safely accessing sideways matrices without the need
+ //to allocate empty matrices as dense, see prepInputMatrices
+ protected static double getValue(double[] data, int index) {
+ return (data!=null) ? data[index] : 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f2ea6336/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java
index 21aabb8..b25d496 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java
@@ -59,12 +59,12 @@ public class MultiAggTmplTest extends AutomatedTestBase
}
@Test
- public void testCodegenRowAgg1CP() {
+ public void testCodegenMultiAgg1CP() {
testCodegenIntegration( TEST_NAME1, false, ExecType.CP );
}
@Test
- public void testCodegenRowAgg1Spark() {
+ public void testCodegenMultiAgg1Spark() {
testCodegenIntegration( TEST_NAME1, false, ExecType.SPARK );
}
@@ -74,12 +74,12 @@ public class MultiAggTmplTest extends AutomatedTestBase
}
@Test
- public void testCodegenRowAgg2CP() {
+ public void testCodegenMultiAgg2CP() {
testCodegenIntegration( TEST_NAME2, false, ExecType.CP );
}
@Test
- public void testCodegenRowAgg2Spark() {
+ public void testCodegenMultiAgg2Spark() {
testCodegenIntegration( TEST_NAME2, false, ExecType.SPARK );
}
[3/3] incubator-systemml git commit: [SYSTEMML-1288] Extended code
generator (multi-agg across partitions)
Posted by mb...@apache.org.
[SYSTEMML-1288] Extended code generator (multi-agg across partitions)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/9820f4c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/9820f4c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/9820f4c5
Branch: refs/heads/master
Commit: 9820f4c5293c69873f68544748507b6473948f12
Parents: 7c15339
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Apr 6 20:57:07 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Apr 6 21:16:31 2017 -0700
----------------------------------------------------------------------
.../sysml/hops/codegen/cplan/CNodeMultiAgg.java | 5 +-
.../template/PlanSelectionFuseCostBased.java | 54 +++++++++++++++++++-
.../hops/codegen/template/TemplateCell.java | 3 +-
3 files changed, 59 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9820f4c5/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeMultiAgg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeMultiAgg.java b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeMultiAgg.java
index 7ec07a6..d9502be 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeMultiAgg.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeMultiAgg.java
@@ -106,7 +106,10 @@ public class CNodeMultiAgg extends CNodeTpl
for( int i=0; i<_outputs.size(); i++ ) {
CNode out = _outputs.get(i);
String tmpOut = getAggTemplate(i);
- tmpOut = tmpOut.replace("%IN%", out.getVarname());
+ //get variable name (w/ handling of direct consumption of inputs)
+ String varName = (out instanceof CNodeData && ((CNodeData)out).getHopID()==
+ ((CNodeData)_inputs.get(0)).getHopID()) ? "a" : out.getVarname();
+ tmpOut = tmpOut.replace("%IN%", varName);
tmpOut = tmpOut.replace("%IX%", String.valueOf(i));
sb.append(tmpOut);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9820f4c5/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java b/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java
index 50d6ff1..151dab2 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java
@@ -34,7 +34,9 @@ import org.apache.commons.logging.LogFactory;
import org.apache.sysml.hops.AggBinaryOp;
import org.apache.sysml.hops.AggUnaryOp;
import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.Hop.AggOp;
import org.apache.sysml.hops.Hop.Direction;
import org.apache.sysml.hops.IndexingOp;
import org.apache.sysml.hops.ParameterizedBuiltinOp;
@@ -82,12 +84,15 @@ public class PlanSelectionFuseCostBased extends PlanSelection
if( LOG.isTraceEnabled() )
LOG.trace("Partition materialization points: "+Arrays.toString(M.toArray(new Long[0])));
- //step 3: create composite templates entries
+ //step 3: create composite templates (within the partition)
createAndAddMultiAggPlans(memo, partition, R);
//step 4: plan enumeration and plan selection
selectPlans(memo, partition, R, M);
}
+
+ //step 5: add composite templates (across partitions)
+ createAndAddMultiAggPlans(memo, roots);
//take all distinct best plans
for( Entry<Long, List<MemoTableEntry>> e : getBestPlans().entrySet() )
@@ -217,6 +222,7 @@ public class PlanSelectionFuseCostBased extends PlanSelection
&& partition.contains(hop.getHopID());
}
+ //within-partition multi-agg templates
private static void createAndAddMultiAggPlans(CPlanMemoTable memo, HashSet<Long> partition, HashSet<Long> R)
{
//create index of plans that reference full aggregates to avoid circular dependencies
@@ -262,6 +268,30 @@ public class PlanSelectionFuseCostBased extends PlanSelection
}
}
+ //across-partition multi-agg templates
+ private static void createAndAddMultiAggPlans(CPlanMemoTable memo, ArrayList<Hop> roots)
+ {
+ //#1: collect full aggregations over shared inputs (otherwise never fused)
+ HashMap<Long, ArrayList<Long>> fullAggs = new HashMap<Long, ArrayList<Long>>();
+ Hop.resetVisitStatus(roots);
+ for( Hop hop : roots )
+ rCollectAggregatesSharedRead(hop, fullAggs);
+
+ //construct and add multiagg template plans (w/ max 3 aggregations)
+ for( Entry<Long, ArrayList<Long>> e : fullAggs.entrySet() ) {
+ if( e.getValue().size()<=1 )
+ continue;
+ ArrayList<Long> aggs = e.getValue();
+ MemoTableEntry me = new MemoTableEntry(TemplateType.MultiAggTpl,
+ aggs.get(0), aggs.get(1), (aggs.size()>2)?aggs.get(2):-1);
+ for( int i=0; i<aggs.size(); i++ ) {
+ memo.add(memo._hopRefs.get(aggs.get(i)), me);
+ if( LOG.isTraceEnabled() )
+ LOG.trace("Added multiagg* plan: "+aggs.get(i)+" "+me);
+ }
+ }
+ }
+
private static boolean isValidMultiAggregate(CPlanMemoTable memo, MemoTableEntry me) {
//ensure that aggregates are independent of each other, i.e.,
//they to not have potentially transitive parent child references
@@ -285,6 +315,28 @@ public class PlanSelectionFuseCostBased extends PlanSelection
return ret;
}
+ private static void rCollectAggregatesSharedRead(Hop current, HashMap<Long, ArrayList<Long>> aggs) {
+ if( current.isVisited() )
+ return;
+
+ //collect all applicable full aggregations per read
+ if( HopRewriteUtils.isAggUnaryOp(current, AggOp.SUM, AggOp.SUM_SQ, AggOp.MIN, AggOp.MAX)
+ && ((AggUnaryOp)current).getDirection()==Direction.RowCol
+ && current.getInput().get(0) instanceof DataOp )
+ {
+ Hop input = current.getInput().get(0);
+ if( !aggs.containsKey(input.getHopID()) )
+ aggs.put(input.getHopID(), new ArrayList<Long>());
+ aggs.get(input.getHopID()).add(current.getHopID());
+ }
+
+ //recursively process children
+ for( Hop c : current.getInput() )
+ rCollectAggregatesSharedRead(c, aggs);
+
+ current.setVisited();
+ }
+
private void selectPlans(CPlanMemoTable memo, HashSet<Long> partition, HashSet<Long> R, ArrayList<Long> M)
{
//if no materialization points, use basic fuse-all w/ partition awareness
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9820f4c5/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
index e3c12d5..885d3db 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
import org.apache.sysml.hops.AggBinaryOp;
import org.apache.sysml.hops.AggUnaryOp;
import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.UnaryOp;
import org.apache.sysml.hops.Hop.AggOp;
@@ -149,7 +150,7 @@ public class TemplateCell extends TemplateBase
MemoTableEntry me = memo.getBest(hop.getHopID(), TemplateType.CellTpl);
for( int i=0; i<hop.getInput().size(); i++ ) {
Hop c = hop.getInput().get(i);
- if( me.isPlanRef(i) )
+ if( me!=null && me.isPlanRef(i) && !(c instanceof DataOp) )
rConstructCplan(c, memo, tmp, inHops, compileLiterals);
else {
CNodeData cdata = TemplateUtils.createCNodeData(c, compileLiterals);