You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/01/16 05:24:33 UTC
[1/3] systemml git commit: [SYSTEMML-2046] Large dense blocks in
rotate180, relu, incl cleanups
Repository: systemml
Updated Branches:
refs/heads/master e7994b745 -> 0a0a40370
[SYSTEMML-2046] Large dense blocks in rotate180, relu, incl cleanups
This patch adds support for large dense blocks in convolution rotate180,
and relu backwards operations. Furthermore, this also includes some
minor cleanups of unnecessary code duplication and inefficiencies.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/185ab0e3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/185ab0e3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/185ab0e3
Branch: refs/heads/master
Commit: 185ab0e3b34ecd8e8281027839a3d53fbacb9e42
Parents: e7994b7
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Jan 15 19:45:48 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Jan 15 19:45:48 2018 -0800
----------------------------------------------------------------------
.../runtime/instructions/InstructionUtils.java | 8 +-
.../runtime/matrix/data/LibMatrixDNNRelu.java | 95 ++++++++++++++----
.../matrix/data/LibMatrixDNNRotate180.java | 34 +++----
.../sysml/runtime/util/ConvolutionUtils.java | 100 -------------------
4 files changed, 93 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/185ab0e3/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
index 91a4546..e6affa0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
@@ -594,10 +594,8 @@ public class InstructionUtils
* @param opcode the opcode
* @param arg1IsScalar ?
* @return scalar operator
- * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static ScalarOperator parseScalarBinaryOperator(String opcode, boolean arg1IsScalar)
- throws DMLRuntimeException
{
//for all runtimes that set constant dynamically (cp/spark)
double default_constant = 0;
@@ -612,10 +610,8 @@ public class InstructionUtils
* @param arg1IsScalar ?
* @param constant ?
* @return scalar operator
- * @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static ScalarOperator parseScalarBinaryOperator(String opcode, boolean arg1IsScalar, double constant)
- throws DMLRuntimeException
{
//commutative operators
if ( opcode.equalsIgnoreCase("+") ){
@@ -754,8 +750,8 @@ public class InstructionUtils
return new LeftScalarOperator(Divide.getDivideFnObject(), constant);
}
- throw new DMLRuntimeException("Unknown binary opcode " + opcode);
- }
+ throw new RuntimeException("Unknown binary opcode " + opcode);
+ }
public static BinaryOperator parseExtendedBinaryOperator(String opcode)
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/systemml/blob/185ab0e3/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRelu.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRelu.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRelu.java
index c44a032..f1c3ecb 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRelu.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRelu.java
@@ -23,17 +23,15 @@ import java.util.concurrent.Callable;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.functionobjects.Plus;
import org.apache.sysml.runtime.instructions.InstructionUtils;
-import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
-import org.apache.sysml.runtime.util.ConvolutionUtils;
+import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
/**
* This class contains the different implementation of rotate180 operation
*/
public class LibMatrixDNNRelu
{
- private static BinaryOperator PLUS = new BinaryOperator(Plus.getPlusFnObject());
+ private static ScalarOperator GT0 = InstructionUtils.parseScalarBinaryOperator(">", false, 0);
/**
@@ -55,35 +53,94 @@ public class LibMatrixDNNRelu
/**
* Performs the operation: (X gt 0) * dout
*/
- public static class ReluBackward implements Callable<Long>
+ public static class ReluBackward implements Callable<Long>
{
- public int _rl; public int _ru;
- private final ConvolutionParameters _params;
- double [] outputArray; int numOutCols;
+ public final int _rl, _ru;
+ private final ConvolutionParameters _params;
public ReluBackward(int rl, int ru, ConvolutionParameters params) {
_rl = rl; _ru = ru;
_params = params;
- outputArray= params.output.getDenseBlockValues();
- numOutCols = params.input1.getNumColumns();
}
@Override
public Long call() throws Exception {
+ //note: X (m x n), dout (m x n) -> out (m x n)
+ DenseBlock out = _params.output.getDenseBlock();
+ final int n = _params.input1.getNumColumns();
if(!_params.input1.isInSparseFormat() && !_params.input2.isInSparseFormat()) {
- double [] inputArr = _params.input1.getDenseBlockValues();
- double [] doutArr = _params.input2.getDenseBlockValues();
- for(int i = _rl*numOutCols; i < _ru*numOutCols; i++) {
- outputArray[i] = inputArr[i] > 0 ? doutArr[i] : 0;
+ DenseBlock x = _params.input1.getDenseBlock();
+ DenseBlock dout = _params.input2.getDenseBlock();
+ for(int i = _rl; i < _ru; i++) {
+ double[] xvals = x.values(i), doutvals = dout.values(i), cvals = out.values(i);
+ int xpos = x.pos(i), doutpos = dout.pos(i), cpos = out.pos(i);
+ for(int j=0; j<n; j++)
+ cvals[cpos+j] = xvals[xpos+j] > 0 ? doutvals[doutpos +j] : 0;
}
}
else {
- // Perform (X > 0)
- ConvolutionUtils.scalarOperations(_params.input1, outputArray, _rl*numOutCols, numOutCols, _rl, _ru,
- InstructionUtils.parseScalarBinaryOperator(">", false, 0));
- // Then perform (X > 0) * dout
- ConvolutionUtils.binaryOperationInPlace(_params.input2, outputArray, _rl*numOutCols, numOutCols, _rl, _ru, PLUS);
+ scalarOperations(_params.input1, out, n, _rl, _ru, GT0); // (X > 0)
+ binaryOperationInPlacePlus(_params.input2, out, n, _rl, _ru); // (X > 0) * dout
}
return 0L;
}
}
+
+ private static void scalarOperations(MatrixBlock src, DenseBlock c,
+ int destNumCols, int src_rl, int src_ru, ScalarOperator op)
+ throws DMLRuntimeException
+ {
+ if(src.isInSparseFormat()) {
+ for(int i = src_rl; i < src_ru; i++) {
+ if( src.getSparseBlock().isEmpty(i) ) continue;
+ int apos = src.getSparseBlock().pos(i);
+ int alen = src.getSparseBlock().size(i);
+ int[] aix = src.getSparseBlock().indexes(i);
+ double[] avals = src.getSparseBlock().values(i);
+ double[] cvals = c.values(i);
+ int cix = c.pos(i);
+ for(int j = apos; j < apos+alen; j++)
+ cvals[ cix+aix[j] ] = op.executeScalar(avals[j]);
+ }
+ }
+ else {
+ DenseBlock a = src.getDenseBlock();
+ for(int i = src_rl; i < src_ru; i++) {
+ double[] avals = a.values(i), cvals = c.values(i);
+ int aix = a.pos(i), cix = c.pos(i);
+ for(int j=0; j<destNumCols; j++)
+ cvals[cix+j] = op.executeScalar(avals[aix+j]);
+ }
+ }
+ }
+
+ private static void binaryOperationInPlacePlus(MatrixBlock src,
+ DenseBlock c, int destNumCols, int src_rl, int src_ru)
+ throws DMLRuntimeException
+ {
+ if( src.isEmptyBlock(false) )
+ return; //do nothing (add 0);
+
+ if(src.isInSparseFormat()) {
+ for(int i = src_rl; i < src_ru; i++) {
+ if( src.getSparseBlock().isEmpty(i) ) continue;
+ int apos = src.getSparseBlock().pos(i);
+ int alen = src.getSparseBlock().size(i);
+ int[] aix = src.getSparseBlock().indexes(i);
+ double[] avals = src.getSparseBlock().values(i);
+ double[] cvals = c.values(i);
+ int cix = c.pos(i);
+ for(int j = apos; j < apos+alen; j++)
+ cvals[ cix+aix[j] ] += avals[j];
+ }
+ }
+ else { //DENSE
+ DenseBlock a = src.getDenseBlock();
+ for(int i = src_rl; i < src_ru; i++) {
+ double[] avals = a.values(i), cvals = c.values(i);
+ int aix = a.pos(i), cix = c.pos(i);
+ for(int j=0; j<destNumCols; j++)
+ cvals[cix+j] += avals[aix+j];
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/185ab0e3/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRotate180.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRotate180.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRotate180.java
index b463794..f4921c4 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRotate180.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNRotate180.java
@@ -29,10 +29,9 @@ public class LibMatrixDNNRotate180
public void execute(int inputN, int outputN);
public static Rotate180Worker getWorker(MatrixBlock in, MatrixBlock out,
ConvolutionParameters params, boolean zeroOutSparseOutput, boolean trans) {
- if(!in.isInSparseFormat())
- return new DenseRotate180Worker(in, out.getDenseBlockValues(), params);
- else
- return new SparseRotate180Worker(in, out, params, trans);
+ return in.isInSparseFormat() ?
+ new SparseRotate180Worker(in, out, params, trans) :
+ new DenseRotate180Worker(in, out, params);
}
}
@@ -40,27 +39,24 @@ public class LibMatrixDNNRotate180
* Performing dense rotate180 (general case)
*/
private static class DenseRotate180Worker implements Rotate180Worker {
- private final double[] inputArray, outputArray;
+ private final DenseBlock in, out;
private final ConvolutionParameters params;
- public DenseRotate180Worker(MatrixBlock input, double[] outputArray, ConvolutionParameters params) {
- this.outputArray = outputArray;
+ public DenseRotate180Worker(MatrixBlock input, MatrixBlock output, ConvolutionParameters params) {
+ this.in = input.getDenseBlock();
+ this.out = output.getDenseBlock();
this.params = params;
- inputArray = input.getDenseBlockValues();
- if(inputArray == null || outputArray == null)
- throw new RuntimeException("Incorrect usage: empty inputs");
}
@Override
public void execute(int inputN, int outputN) {
- int outputOffset = outputN*params.K*params.P*params.Q;
- for (int k = 0; k < params.K; k++) {
- for (int p = 0; p < params.P; p++) {
- for (int q = 0; q < params.Q; q++) {
- outputArray[outputOffset + p*params.Q*params.K + q*params.K + k] =
- inputArray[inputN*params.K*params.P*params.Q + k*params.P*params.Q + p*params.Q + q];
- }
- }
- }
+ //note: in (m x KPQ) -> out (m x KPQ)
+ double[] avals = in.values(inputN), cvals = out.values(outputN);
+ int aix = in.pos(inputN), cix = out.pos(outputN);
+ int K = params.K, P = params.P, Q = params.Q;
+ for (int k = 0; k < K; k++)
+ for (int p = 0; p < P; p++)
+ for (int q = 0; q < Q; q++)
+ cvals[cix + p*Q*K + q*K + k] = avals[aix + k*P*Q + p*Q + q];
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/185ab0e3/src/main/java/org/apache/sysml/runtime/util/ConvolutionUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/ConvolutionUtils.java b/src/main/java/org/apache/sysml/runtime/util/ConvolutionUtils.java
index 87f1240..9800773 100644
--- a/src/main/java/org/apache/sysml/runtime/util/ConvolutionUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/util/ConvolutionUtils.java
@@ -22,11 +22,7 @@ package org.apache.sysml.runtime.util;
import java.util.Arrays;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.functionobjects.Multiply;
-import org.apache.sysml.runtime.functionobjects.Plus;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
-import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
public class ConvolutionUtils {
@@ -73,102 +69,6 @@ public class ConvolutionUtils {
}
return ret;
}
-
-
- // Performs dest[destPos...] op= thatValue[src_rl:src_ru,]
- public static void binaryOperationInPlace(MatrixBlock src, double [] dest,
- int destPos, int destNumCols, int src_rl, int src_ru, BinaryOperator op) throws DMLRuntimeException {
- if(src.isInSparseFormat()) {
- if(src.isEmptyBlock() && op.fn == Plus.getPlusFnObject()) {
- // Do nothing: Inplace addition by zero
- }
- else if(src.isEmptyBlock() && op.fn == Multiply.getMultiplyFnObject()) {
- // Inplace multiplication by zero
- Arrays.fill(dest, destPos, destPos + (src_ru-src_rl)*destNumCols, 0);
- }
- else if(op.fn == Plus.getPlusFnObject()) {
- for(int i = src_rl, cix = destPos; i < src_ru; i++, cix += destNumCols) {
- if( !src.getSparseBlock().isEmpty(i) ) {
- int apos = src.getSparseBlock().pos(i);
- int alen = src.getSparseBlock().size(i);
- int[] aix = src.getSparseBlock().indexes(i);
- double[] avals = src.getSparseBlock().values(i);
- for(int j = apos; j < apos+alen; j++) {
- dest[ cix+aix[j] ] += avals[j];
- }
- }
- }
- }
- else if(op.fn == Multiply.getMultiplyFnObject()) {
- // Unsafe operation
- for(int i = src_rl, cix = destPos; i < src_ru; i++, cix += destNumCols) {
- if( !src.getSparseBlock().isEmpty(i) ) {
- int apos = src.getSparseBlock().pos(i);
- int alen = src.getSparseBlock().size(i);
- int[] aix = src.getSparseBlock().indexes(i);
- double[] avals = src.getSparseBlock().values(i);
- int prevDestIndex = 0;
- for(int j = apos; j < apos+alen; j++) {
- // Multiplication by zero. Assumption: aix is sorted.
- Arrays.fill(dest, cix+prevDestIndex, cix+aix[j], 0);
- prevDestIndex = aix[j]+1;
- dest[ cix+aix[j] ] *= avals[j];
- }
- Arrays.fill(dest, cix+prevDestIndex, cix+destNumCols, 0);
- }
- else {
- Arrays.fill(dest, cix, cix + destNumCols, 0);
- }
- }
- }
- else {
- // As operation could be safe or unsafe. This will be caught at development time.
- throw new DMLRuntimeException("Unimplemented sparse operation");
- }
- }
- else {
- double [] inputArr = src.getDenseBlockValues();
- if(op.fn == Plus.getPlusFnObject()) {
- for(int i = destPos; i < src_ru*destNumCols; i++) {
- dest[i] += inputArr[i];
- }
- }
- else if(op.fn == Multiply.getMultiplyFnObject()) {
- for(int i = destPos; i < src_ru*destNumCols; i++) {
- dest[i] *= inputArr[i];
- }
- }
- else {
- for(int i = destPos; i < src_ru*destNumCols; i++) {
- dest[i] = op.fn.execute(dest[i], inputArr[i]);
- }
- }
- }
- }
-
- // Performs dest[destPos...] = src[src_rl:src_ru,] op scalar
- public static void scalarOperations(MatrixBlock src, double [] dest,
- int destPos, int destNumCols, int src_rl, int src_ru, ScalarOperator scalarOp) throws DMLRuntimeException {
- if(src.isInSparseFormat()) {
- for(int i = src_rl, cix = destPos; i < src_ru; i++, cix += destNumCols) {
- if( !src.getSparseBlock().isEmpty(i) ) {
- int apos = src.getSparseBlock().pos(i);
- int alen = src.getSparseBlock().size(i);
- int[] aix = src.getSparseBlock().indexes(i);
- double[] avals = src.getSparseBlock().values(i);
- for(int j = apos; j < apos+alen; j++) {
- dest[ cix+aix[j] ] = scalarOp.executeScalar(avals[j]);
- }
- }
- }
- }
- else {
- double [] inputArr = src.getDenseBlockValues();
- for(int i = destPos; i < src_ru*destNumCols; i++) {
- dest[i] = scalarOp.executeScalar(inputArr[i]);
- }
- }
- }
public static void fillBias(MatrixBlock bias, double [] outputArray, int src_rl, int src_ru, int N, int K, int PQ) throws DMLRuntimeException {
// bias.getNumColumns() == 1 checked outside
[3/3] systemml git commit: [MINOR] Performance nrow/ncol runtime
instructions (parsing overhead)
Posted by mb...@apache.org.
[MINOR] Performance nrow/ncol runtime instructions (parsing overhead)
This patch makes a minor performance improvement to the very common nrow
and ncol instructions by (1) avoiding repeated opcode string comparisons
on instruction execution, and (2) cleaning up the unnecessarily
convoluted code path. On a multi-threaded scenario of 240,000,000 nrow
operations, this patch improved the total runtime from 88.5 to 65.2s
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/0a0a4037
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/0a0a4037
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/0a0a4037
Branch: refs/heads/master
Commit: 0a0a403703237be548510f98c2c871da72452e0c
Parents: 151f658
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Jan 15 20:58:06 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Jan 15 20:58:06 2018 -0800
----------------------------------------------------------------------
.../cp/AggregateUnaryCPInstruction.java | 94 +++++++++-----------
.../cp/CentralMomentCPInstruction.java | 2 +-
2 files changed, 45 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/0a0a4037/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
index d57a8dd..c6e3cce 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
@@ -35,34 +35,44 @@ import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
import org.apache.sysml.runtime.matrix.operators.Operator;
import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
-public class AggregateUnaryCPInstruction extends UnaryCPInstruction {
-
- private AggregateUnaryCPInstruction(Operator op, CPOperand in, CPOperand out, String opcode, String istr) {
- this(op, in, null, null, out, opcode, istr);
+public class AggregateUnaryCPInstruction extends UnaryCPInstruction
+{
+ public enum AUType {
+ NROW, NCOL, LENGTH,
+ DEFAULT;
+ public boolean isMeta() {
+ return this != DEFAULT;
+ }
+ }
+
+ private final AUType _type;
+
+ private AggregateUnaryCPInstruction(Operator op, CPOperand in, CPOperand out, AUType type, String opcode, String istr) {
+ this(op, in, null, null, out, type, opcode, istr);
}
protected AggregateUnaryCPInstruction(Operator op, CPOperand in1, CPOperand in2, CPOperand in3, CPOperand out,
- String opcode, String istr) {
+ AUType type, String opcode, String istr) {
super(CPType.AggregateUnary, op, in1, in2, in3, out, opcode, istr);
+ _type = type;
}
-
+
public static AggregateUnaryCPInstruction parseInstruction(String str)
throws DMLRuntimeException
{
String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
- String opcode = parts[0];
+ String opcode = parts[0];
CPOperand in1 = new CPOperand(parts[1]);
CPOperand out = new CPOperand(parts[2]);
if(opcode.equalsIgnoreCase("nrow") || opcode.equalsIgnoreCase("ncol") || opcode.equalsIgnoreCase("length")){
return new AggregateUnaryCPInstruction(new SimpleOperator(Builtin.getBuiltinFnObject(opcode)),
- in1, out, opcode, str);
+ in1, out, AUType.valueOf(opcode.toUpperCase()), opcode, str);
}
- else //DEFAULT BEHAVIOR
- {
+ else { //DEFAULT BEHAVIOR
AggregateUnaryOperator aggun = InstructionUtils
.parseBasicAggregateUnaryOperator(opcode, Integer.parseInt(parts[3]));
- return new AggregateUnaryCPInstruction(aggun, in1, out, opcode, str);
+ return new AggregateUnaryCPInstruction(aggun, in1, out, AUType.DEFAULT, opcode, str);
}
}
@@ -73,22 +83,15 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction {
String output_name = output.getName();
String opcode = getOpcode();
- if( opcode.equalsIgnoreCase("nrow") || opcode.equalsIgnoreCase("ncol") || opcode.equalsIgnoreCase("length") )
+ if( _type.isMeta() ) //nrow/ncol/length
{
//check existence of input variable
- if( !ec.getVariables().keySet().contains(input1.getName()) ){
+ if( !ec.getVariables().keySet().contains(input1.getName()) )
throw new DMLRuntimeException("Variable '"+input1.getName()+"' does not exist.");
- }
//get meta data information
MatrixCharacteristics mc = ec.getMatrixCharacteristics(input1.getName());
- long rval = -1;
- if(opcode.equalsIgnoreCase("nrow"))
- rval = mc.getRows();
- else if(opcode.equalsIgnoreCase("ncol"))
- rval = mc.getCols();
- else if(opcode.equalsIgnoreCase("length"))
- rval = mc.getRows() * mc.getCols();
+ long rval = getSizeMetaData(_type, mc);
//check for valid output, and acquire read if necessary
//(Use case: In case of forced exec type singlenode, there are no reblocks. For csv
@@ -97,12 +100,11 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction {
//Note: check on matrix characteristics to cover incorrect length (-1*-1 -> 1)
if( !mc.dimsKnown() ) //invalid nrow/ncol/length
{
- if( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE
+ if( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE
|| (input1.getDataType() == DataType.FRAME && OptimizerUtils.isHadoopExecutionMode()) )
{
- if( OptimizerUtils.isHadoopExecutionMode() ) {
+ if( OptimizerUtils.isHadoopExecutionMode() )
LOG.warn("Reading csv input frame of unkown size into memory for '"+opcode+"'.");
- }
//read the input matrix/frame and explicitly refresh meta data
CacheableData<?> obj = ec.getCacheableData(input1.getName());
@@ -112,12 +114,7 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction {
//update meta data information
mc = ec.getMatrixCharacteristics(input1.getName());
- if(opcode.equalsIgnoreCase("nrow"))
- rval = mc.getRows();
- else if(opcode.equalsIgnoreCase("ncol"))
- rval = mc.getCols();
- else if(opcode.equalsIgnoreCase("length"))
- rval = mc.getRows() * mc.getCols();
+ rval = getSizeMetaData(_type, mc);
}
else {
throw new DMLRuntimeException("Invalid meta data returned by '"+opcode+"': "+rval + ":" + instString);
@@ -125,36 +122,33 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction {
}
//create and set output scalar
- ScalarObject ret = null;
- switch( output.getValueType() ) {
- case INT: ret = new IntObject(rval); break;
- case DOUBLE: ret = new DoubleObject(rval); break;
- case STRING: ret = new StringObject(String.valueOf(rval)); break;
-
- default:
- throw new DMLRuntimeException("Invalid output value type: "+output.getValueType());
- }
- ec.setScalarOutput(output_name, ret);
- return;
+ ec.setScalarOutput(output_name, new IntObject(rval));
}
- else
- {
- /* Default behavior for AggregateUnary Instruction */
- MatrixBlock matBlock = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
+ else { //DEFAULT
+ MatrixBlock matBlock = ec.getMatrixInput(input1.getName());
AggregateUnaryOperator au_op = (AggregateUnaryOperator) _optr;
- MatrixBlock resultBlock = (MatrixBlock) matBlock.aggregateUnaryOperations(au_op, new MatrixBlock(), matBlock.getNumRows(), matBlock.getNumColumns(), new MatrixIndexes(1, 1), true);
-
- ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
+ MatrixBlock resultBlock = (MatrixBlock) matBlock.aggregateUnaryOperations(au_op, new MatrixBlock(),
+ matBlock.getNumRows(), matBlock.getNumColumns(), new MatrixIndexes(1, 1), true);
+ ec.releaseMatrixInput(input1.getName());
if(output.getDataType() == DataType.SCALAR){
DoubleObject ret = new DoubleObject(resultBlock.getValue(0, 0));
ec.setScalarOutput(output_name, ret);
} else{
// since the computed value is a scalar, allocate a "temp" output matrix
- ec.setMatrixOutput(output_name, resultBlock, getExtendedOpcode());
+ ec.setMatrixOutput(output_name, resultBlock);
}
}
}
-
+
+ private static long getSizeMetaData(AUType type, MatrixCharacteristics mc) {
+ switch( type ) {
+ case NROW: return mc.getRows();
+ case NCOL: return mc.getCols();
+ case LENGTH: return mc.getRows() * mc.getCols();
+ default:
+ throw new RuntimeException("Opcode not applicable: "+type.name());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/0a0a4037/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java
index c13ca4c..1bd5b60 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CentralMomentCPInstruction.java
@@ -33,7 +33,7 @@ public class CentralMomentCPInstruction extends AggregateUnaryCPInstruction {
private CentralMomentCPInstruction(CMOperator cm, CPOperand in1, CPOperand in2, CPOperand in3, CPOperand out,
String opcode, String str) {
- super(cm, in1, in2, in3, out, opcode, str);
+ super(cm, in1, in2, in3, out, AUType.DEFAULT, opcode, str);
}
public static CentralMomentCPInstruction parseInstruction(String str)
[2/3] systemml git commit: [SYSTEMML-2073] Fix lost update on
lock-free statistics maintenance
Posted by mb...@apache.org.
[SYSTEMML-2073] Fix lost update on lock-free statistics maintenance
The recently introduced lock-free statistics maintenance (which
significantly reduced contention for high frequency updates), had a
problem of lost update in case multiple threads try to insert a missing
opcode. This patch hardens the lock-free update for such cases without
affecting the common case of existing opcodes.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/151f658c
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/151f658c
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/151f658c
Branch: refs/heads/master
Commit: 151f658cd071aea06987eef08596ec179e4eb9c0
Parents: 185ab0e
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Jan 15 20:27:02 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Jan 15 20:27:02 2018 -0800
----------------------------------------------------------------------
src/main/java/org/apache/sysml/utils/Statistics.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/151f658c/src/main/java/org/apache/sysml/utils/Statistics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/Statistics.java b/src/main/java/org/apache/sysml/utils/Statistics.java
index 414f0e7..c003f30 100644
--- a/src/main/java/org/apache/sysml/utils/Statistics.java
+++ b/src/main/java/org/apache/sysml/utils/Statistics.java
@@ -533,11 +533,12 @@ public class Statistics
* @param timeNanos time in nano seconds
*/
public static void maintainCPHeavyHitters( String instName, long timeNanos ) {
- //maintain instruction entry
+ //maintain instruction entry (w/ robustness for concurrent updates)
InstStats tmp = _instStats.get(instName);
if( tmp == null ) {
- tmp = new InstStats();
- _instStats.put(instName, tmp);
+ InstStats tmp0 = new InstStats();
+ InstStats tmp1 = _instStats.putIfAbsent(instName, tmp0);
+ tmp = (tmp1 != null) ? tmp1 : tmp0;
}
//thread-local maintenance of instruction stats