You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/11/30 21:00:02 UTC
[1/2] systemml git commit: [SYSTEMML-2031] Fix robustness compression
w/ incompressible data
Repository: systemml
Updated Branches:
refs/heads/master aefab8f8c -> d90073d80
[SYSTEMML-2031] Fix robustness compression w/ incompressible data
This patch improves the robustness of matrix compression by returning
the uncompressed block if all columns are incompressible or the
compression ratio is below 1. This avoids unnecessary size and
computation overhead for an unnecessary indirection to a single
uncompressed column group.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/78586a13
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/78586a13
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/78586a13
Branch: refs/heads/master
Commit: 78586a13114b9fed585060e2ab8976ba6f9b50bd
Parents: aefab8f
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Nov 29 20:28:31 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Wed Nov 29 22:30:53 2017 -0800
----------------------------------------------------------------------
.../runtime/compress/CompressedMatrixBlock.java | 24 ++++++++++--
.../cp/CompressionCPInstruction.java | 8 ++--
.../spark/CompressionSPInstruction.java | 41 +++++++-------------
.../sysml/runtime/matrix/data/MatrixBlock.java | 3 +-
4 files changed, 41 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/78586a13/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
index 58d008a..e1a6a7b 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
@@ -214,21 +214,23 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
* +per column sparsity
*
* @throws DMLRuntimeException if DMLRuntimeException occurs
+ * @return compressed matrix block or original block if incompressible
*/
- public void compress()
+ public MatrixBlock compress()
throws DMLRuntimeException
{
//default sequential execution
- compress(1);
+ return compress(1);
}
/**
* Compress block.
*
* @param k number of threads
+ * @return compressed matrix block or original block if incompressible
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
- public void compress(int k)
+ public MatrixBlock compress(int k)
throws DMLRuntimeException
{
//check for redundant compression
@@ -304,7 +306,13 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
LOG.debug("Compression statistics:");
LOG.debug("--compression phase 1: "+_stats.timePhase1);
}
-
+
+ if( colsC.isEmpty() ) {
+ if( LOG.isDebugEnabled() )
+ LOG.debug("Abort block compression because all columns are incompressible.");
+ return new MatrixBlock().copyShallow(this);
+ }
+
// PHASE 2: Grouping columns
// Divide the bitmap columns into column groups.
List<int[]> bitmapColGrps = PlanningCoCoder.findCocodesByPartitioning(
@@ -366,6 +374,12 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
_stats.size = estimateCompressedSizeInMemory();
_stats.ratio= estimateSizeInMemory() / _stats.size;
+ if( _stats.ratio < 1 ) {
+ if( LOG.isDebugEnabled() )
+ LOG.debug("Abort block compression because compression ratio is less than 1.");
+ return new MatrixBlock().copyShallow(this);
+ }
+
//final cleanup (discard uncompressed block)
rawblock.cleanupBlock(true, true);
this.cleanupBlock(true, true);
@@ -382,6 +396,8 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
LOG.debug("--compressed size: "+_stats.size);
LOG.debug("--compression ratio: "+_stats.ratio);
}
+
+ return this;
}
public CompressionStatistics getCompressionStatistics() {
http://git-wip-us.apache.org/repos/asf/systemml/blob/78586a13/src/main/java/org/apache/sysml/runtime/instructions/cp/CompressionCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CompressionCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CompressionCPInstruction.java
index d9f6c32..630cdc2 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CompressionCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CompressionCPInstruction.java
@@ -38,7 +38,7 @@ public class CompressionCPInstruction extends UnaryCPInstruction {
throws DMLRuntimeException
{
String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
- String opcode = parts[0];
+ String opcode = parts[0];
CPOperand in1 = new CPOperand(parts[1]);
CPOperand out = new CPOperand(parts[2]);
@@ -53,11 +53,11 @@ public class CompressionCPInstruction extends UnaryCPInstruction {
MatrixBlock in = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
//compress the matrix block
- CompressedMatrixBlock cmb = new CompressedMatrixBlock(in);
- cmb.compress(OptimizerUtils.getConstrainedNumThreads(-1));
+ MatrixBlock out = new CompressedMatrixBlock(in)
+ .compress(OptimizerUtils.getConstrainedNumThreads(-1));
//set output and release input
ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
- ec.setMatrixOutput(output.getName(), cmb, getExtendedOpcode());
+ ec.setMatrixOutput(output.getName(), out, getExtendedOpcode());
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/78586a13/src/main/java/org/apache/sysml/runtime/instructions/spark/CompressionSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CompressionSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CompressionSPInstruction.java
index ac93266..cdf3f26 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CompressionSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CompressionSPInstruction.java
@@ -35,53 +35,42 @@ public class CompressionSPInstruction extends UnarySPInstruction {
private CompressionSPInstruction(Operator op, CPOperand in, CPOperand out, String opcode, String istr) {
super(op, in, out, opcode, istr);
- _sptype = SPINSTRUCTION_TYPE.Reorg;
+ _sptype = SPINSTRUCTION_TYPE.Compression;
}
public static CompressionSPInstruction parseInstruction ( String str )
- throws DMLRuntimeException
+ throws DMLRuntimeException
{
InstructionUtils.checkNumFields(str, 2);
String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
-
- String opcode = parts[0];
- CPOperand in = new CPOperand(parts[1]);
- CPOperand out = new CPOperand(parts[2]);
-
- return new CompressionSPInstruction(null, in, out, opcode, str);
+ return new CompressionSPInstruction(null,
+ new CPOperand(parts[1]), new CPOperand(parts[2]), parts[0], str);
}
@Override
public void processInstruction(ExecutionContext ec)
- throws DMLRuntimeException
+ throws DMLRuntimeException
{
SparkExecutionContext sec = (SparkExecutionContext)ec;
-
+
//get input rdd handle
- JavaPairRDD<MatrixIndexes,MatrixBlock> in =
- sec.getBinaryBlockRDDHandleForVariable( input1.getName() );
-
+ JavaPairRDD<MatrixIndexes,MatrixBlock> in =
+ sec.getBinaryBlockRDDHandleForVariable( input1.getName() );
+
//execute compression
- JavaPairRDD<MatrixIndexes,MatrixBlock> out =
- in.mapValues(new CompressionFunction());
-
+ JavaPairRDD<MatrixIndexes,MatrixBlock> out =
+ in.mapValues(new CompressionFunction());
+
//set outputs
sec.setRDDHandleForVariable(output.getName(), out);
sec.addLineageRDD(input1.getName(), output.getName());
}
- public static class CompressionFunction implements Function<MatrixBlock,MatrixBlock>
- {
+ public static class CompressionFunction implements Function<MatrixBlock,MatrixBlock> {
private static final long serialVersionUID = -6528833083609423922L;
-
@Override
- public MatrixBlock call(MatrixBlock arg0)
- throws Exception
- {
- CompressedMatrixBlock cmb = new CompressedMatrixBlock(arg0);
- cmb.compress();
-
- return cmb;
+ public MatrixBlock call(MatrixBlock arg0) throws Exception {
+ return new CompressedMatrixBlock(arg0).compress();
}
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/78586a13/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 22b20e4..c893bc6 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -1349,7 +1349,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
copyDenseToDense(that);
}
- public void copyShallow(MatrixBlock that) {
+ public MatrixBlock copyShallow(MatrixBlock that) {
rlen = that.rlen;
clen = that.clen;
nonZeros = that.nonZeros;
@@ -1358,6 +1358,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
denseBlock = that.denseBlock;
else
sparseBlock = that.sparseBlock;
+ return this;
}
private void copySparseToSparse(MatrixBlock that)
[2/2] systemml git commit: [SYSTEMML-1990] New cleanup rewrite for
removal of empty basic blocks
Posted by mb...@apache.org.
[SYSTEMML-1990] New cleanup rewrite for removal of empty basic blocks
This new rewrite removes empty basic blocks, i.e., last-level blocks
without hops, which can originate from the original program or a
sequence of rewrites.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d90073d8
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d90073d8
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d90073d8
Branch: refs/heads/master
Commit: d90073d804322f6b601b0f06d03979dae6322c25
Parents: 78586a1
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Nov 29 22:30:30 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Wed Nov 29 22:30:54 2017 -0800
----------------------------------------------------------------------
.../sysml/hops/rewrite/ProgramRewriter.java | 3 +-
.../rewrite/RewriteRemoveEmptyBasicBlocks.java | 58 ++++++++++++++++++++
2 files changed, 60 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d90073d8/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
index ee8d27c..a73ea6d 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
@@ -105,7 +105,7 @@ public class ProgramRewriter
if( OptimizerUtils.ALLOW_BRANCH_REMOVAL ) {
_sbRuleSet.add( new RewriteRemoveUnnecessaryBranches() ); //dependency: constant folding
_sbRuleSet.add( new RewriteMergeBlockSequence() ); //dependency: remove branches
- }
+ }
_sbRuleSet.add( new RewriteCompressedReblock() );
if( OptimizerUtils.ALLOW_SPLIT_HOP_DAGS )
_sbRuleSet.add( new RewriteSplitDagUnknownCSVRead() ); //dependency: reblock, merge blocks
@@ -137,6 +137,7 @@ public class ProgramRewriter
_dagRuleSet.add( new RewriteRemoveUnnecessaryCasts() );
if( OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION )
_dagRuleSet.add( new RewriteCommonSubexpressionElimination(true) );
+ _sbRuleSet.add( new RewriteRemoveEmptyBasicBlocks() );
}
/**
http://git-wip-us.apache.org/repos/asf/systemml/blob/d90073d8/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveEmptyBasicBlocks.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveEmptyBasicBlocks.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveEmptyBasicBlocks.java
new file mode 100644
index 0000000..5010618
--- /dev/null
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveEmptyBasicBlocks.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.hops.rewrite;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.parser.StatementBlock;
+
+/**
+ * Rule: Simplify program structure by removing empty last-level blocks,
+ * which may originate from the original program or due to a sequence of
+ * rewrites (e.g., checkpoint injection and subsequent IPA).
+ */
+public class RewriteRemoveEmptyBasicBlocks extends StatementBlockRewriteRule
+{
+ @Override
+ public List<StatementBlock> rewriteStatementBlock(StatementBlock sb, ProgramRewriteStatus state)
+ throws HopsException
+ {
+ ArrayList<StatementBlock> ret = new ArrayList<>();
+
+ //prune last level blocks with empty hops
+ if( HopRewriteUtils.isLastLevelStatementBlock(sb)
+ && (sb.getHops() == null || sb.getHops().isEmpty()) ) {
+ if( LOG.isDebugEnabled() )
+ LOG.debug("Applied removeEmptyBasicBlocks (lines "+sb.getBeginLine()+"-"+sb.getEndLine()+").");
+ }
+ else //keep original sb
+ ret.add( sb );
+
+ return ret;
+ }
+
+ @Override
+ public List<StatementBlock> rewriteStatementBlocks(List<StatementBlock> sbs, ProgramRewriteStatus sate)
+ throws HopsException {
+ return sbs;
+ }
+}