You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/02/18 09:40:59 UTC
systemml git commit: [MINOR] Fine-tuning codegen row ops (load
balance, nnz recompute)
Repository: systemml
Updated Branches:
refs/heads/master 5e3fed25b -> f4954b78f
[MINOR] Fine-tuning codegen row ops (load balance, nnz recompute)
This patch addresses minor issues of unnecessary many tasks (which is
unnecessary for small inputs) and redundant nnz recomputation in spark
codegen row ops.
Furthermore, this also includes a minor fix of the maintenance of
codegen optimization stats, which did not properly account for the plans
evaluated by the opening heuristic.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f4954b78
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f4954b78
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f4954b78
Branch: refs/heads/master
Commit: f4954b78f74cf494e61b8798709c26d411913d03
Parents: 5e3fed2
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Feb 17 18:21:28 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Feb 17 21:16:31 2018 -0800
----------------------------------------------------------------------
.../sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java | 2 +-
.../java/org/apache/sysml/runtime/codegen/SpoofRowwise.java | 8 +++++---
.../sysml/runtime/instructions/spark/SpoofSPInstruction.java | 1 +
3 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/f4954b78/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
index 20cb955..84e4b4c 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
@@ -227,7 +227,7 @@ public class PlanSelectionFuseCostBasedV2 extends PlanSelection
//bestC is monotonically non-increasing and serves as the upper bound
final int Mlen = matPoints.length-off;
final long len = UtilFunctions.pow(2, Mlen);
- long numEvalPlans = 0, numEvalPartPlans = 0;
+ long numEvalPlans = 2, numEvalPartPlans = 0;
//evaluate heuristics fuse-all and fuse-no-redundancy to quickly obtain a good lower bound
final boolean[] plan0 = createAssignment(Mlen, off, 0); // fuse-all
http://git-wip-us.apache.org/repos/asf/systemml/blob/f4954b78/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java
index b026b46..f10f857 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java
@@ -163,14 +163,15 @@ public abstract class SpoofRowwise extends SpoofOperator
//post-processing
if( allocTmp &&_reqVectMem > 0 )
LibSpoofPrimitives.cleanupThreadLocalMemory();
- out.recomputeNonZeros();
if( flipOut ) {
fixTransposeDimensions(out);
out = LibMatrixReorg.transpose(out, new MatrixBlock(
out.getNumColumns(), out.getNumRows(), false));
}
- if( !aggIncr )
+ if( !aggIncr ) {
+ out.recomputeNonZeros();
out.examSparsity();
+ }
return out;
}
@@ -207,7 +208,7 @@ public abstract class SpoofRowwise extends SpoofOperator
ExecutorService pool = Executors.newFixedThreadPool( k );
ArrayList<Integer> blklens = (a instanceof CompressedMatrixBlock) ?
LibMatrixMult.getAlignedBlockSizes(m, k, BitmapEncoder.BITMAP_BLOCK_SZ) :
- LibMatrixMult.getBalancedBlockSizesDefault(m, k, false);
+ LibMatrixMult.getBalancedBlockSizesDefault(m, k, (long)m*n<16*PAR_NUMCELL_THRESHOLD);
try
{
@@ -287,6 +288,7 @@ public abstract class SpoofRowwise extends SpoofOperator
int rlen = out.getNumRows();
out.setNumRows(out.getNumColumns());
out.setNumColumns(rlen);
+ out.setNonZeros(out.getNumRows()*out.getNumColumns());
}
private void executeDense(DenseBlock a, SideInput[] b, double[] scalars, DenseBlock c, int n, int rl, int ru) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/f4954b78/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java
index 82b34df..edd7418 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java
@@ -478,6 +478,7 @@ public class SpoofSPInstruction extends SPInstruction {
//cleanup and final result preparations
LibSpoofPrimitives.cleanupThreadLocalMemory();
if( aggIncr ) {
+ blkOut.recomputeNonZeros();
blkOut.examSparsity(); //deferred format change
ret.add(new Tuple2<>(new MatrixIndexes(1,1), blkOut));
}