You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ac...@apache.org on 2016/06/22 04:41:23 UTC
[2/3] incubator-systemml git commit: [SYSTEMML-562] Frame Left
Indexing
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
index c021c04..1953e99 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
@@ -51,7 +51,7 @@ import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.instructions.cp.DoubleObject;
import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
-import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcast;
import org.apache.sysml.runtime.instructions.spark.functions.FilterNonEmptyBlocksFunction;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -237,8 +237,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
|| WeightedCrossEntropy.OPCODE.equalsIgnoreCase(getOpcode())
|| WeightedUnaryMM.OPCODE.equalsIgnoreCase(getOpcode()))
{
- PartitionedBroadcastMatrix bc1 = sec.getBroadcastForVariable( input2.getName() );
- PartitionedBroadcastMatrix bc2 = sec.getBroadcastForVariable( input3.getName() );
+ PartitionedBroadcast<MatrixBlock> bc1 = sec.getBroadcastForVariable( input2.getName() );
+ PartitionedBroadcast<MatrixBlock> bc2 = sec.getBroadcastForVariable( input3.getName() );
//partitioning-preserving mappartitions (key access required for broadcast loopkup)
boolean noKeyChange = (qop.wtype3 == null || qop.wtype3.isBasic()); //only wdivmm changes keys
@@ -251,8 +251,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
//reduce-side operation (two/three/four rdd inputs, zero/one/two broadcasts)
else
{
- PartitionedBroadcastMatrix bc1 = _cacheU ? sec.getBroadcastForVariable( input2.getName() ) : null;
- PartitionedBroadcastMatrix bc2 = _cacheV ? sec.getBroadcastForVariable( input3.getName() ) : null;
+ PartitionedBroadcast<MatrixBlock> bc1 = _cacheU ? sec.getBroadcastForVariable( input2.getName() ) : null;
+ PartitionedBroadcast<MatrixBlock> bc2 = _cacheV ? sec.getBroadcastForVariable( input3.getName() ) : null;
JavaPairRDD<MatrixIndexes,MatrixBlock> inU = (!_cacheU) ? sec.getBinaryBlockRDDHandleForVariable( input2.getName() ) : null;
JavaPairRDD<MatrixIndexes,MatrixBlock> inV = (!_cacheV) ? sec.getBinaryBlockRDDHandleForVariable( input3.getName() ) : null;
JavaPairRDD<MatrixIndexes,MatrixBlock> inW = (qop.hasFourInputs() && !_input4.isLiteral()) ?
@@ -361,10 +361,10 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
private static final long serialVersionUID = -3175397651350954930L;
protected QuaternaryOperator _qop = null;
- protected PartitionedBroadcastMatrix _pmU = null;
- protected PartitionedBroadcastMatrix _pmV = null;
+ protected PartitionedBroadcast<MatrixBlock> _pmU = null;
+ protected PartitionedBroadcast<MatrixBlock> _pmV = null;
- public RDDQuaternaryBaseFunction( QuaternaryOperator qop, PartitionedBroadcastMatrix bcU, PartitionedBroadcastMatrix bcV ) {
+ public RDDQuaternaryBaseFunction( QuaternaryOperator qop, PartitionedBroadcast<MatrixBlock> bcU, PartitionedBroadcast<MatrixBlock> bcV ) {
_qop = qop;
_pmU = bcU;
_pmV = bcV;
@@ -388,7 +388,7 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
{
private static final long serialVersionUID = -8209188316939435099L;
- public RDDQuaternaryFunction1( QuaternaryOperator qop, PartitionedBroadcastMatrix bcU, PartitionedBroadcastMatrix bcV )
+ public RDDQuaternaryFunction1( QuaternaryOperator qop, PartitionedBroadcast<MatrixBlock> bcU, PartitionedBroadcast<MatrixBlock> bcV )
throws DMLRuntimeException
{
super(qop, bcU, bcV);
@@ -415,8 +415,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
MatrixBlock blkIn = arg._2();
MatrixBlock blkOut = new MatrixBlock();
- MatrixBlock mbU = _pmU.getMatrixBlock((int)ixIn.getRowIndex(), 1);
- MatrixBlock mbV = _pmV.getMatrixBlock((int)ixIn.getColumnIndex(), 1);
+ MatrixBlock mbU = _pmU.getBlock((int)ixIn.getRowIndex(), 1);
+ MatrixBlock mbV = _pmV.getBlock((int)ixIn.getColumnIndex(), 1);
//execute core operation
blkIn.quaternaryOperations(_qop, mbU, mbV, null, blkOut);
@@ -437,7 +437,7 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
{
private static final long serialVersionUID = 7493974462943080693L;
- public RDDQuaternaryFunction2( QuaternaryOperator qop, PartitionedBroadcastMatrix bcU, PartitionedBroadcastMatrix bcV )
+ public RDDQuaternaryFunction2( QuaternaryOperator qop, PartitionedBroadcast<MatrixBlock> bcU, PartitionedBroadcast<MatrixBlock> bcV )
throws DMLRuntimeException
{
super(qop, bcU, bcV);
@@ -452,8 +452,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
MatrixBlock blkIn2 = arg0._2()._2();
MatrixBlock blkOut = new MatrixBlock();
- MatrixBlock mbU = (_pmU!=null)?_pmU.getMatrixBlock((int)ixIn.getRowIndex(), 1) : blkIn2;
- MatrixBlock mbV = (_pmV!=null)?_pmV.getMatrixBlock((int)ixIn.getColumnIndex(), 1) : blkIn2;
+ MatrixBlock mbU = (_pmU!=null)?_pmU.getBlock((int)ixIn.getRowIndex(), 1) : blkIn2;
+ MatrixBlock mbV = (_pmV!=null)?_pmV.getBlock((int)ixIn.getColumnIndex(), 1) : blkIn2;
MatrixBlock mbW = (_qop.hasFourInputs()) ? blkIn2 : null;
//execute core operation
@@ -473,7 +473,7 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
{
private static final long serialVersionUID = -2294086455843773095L;
- public RDDQuaternaryFunction3( QuaternaryOperator qop, PartitionedBroadcastMatrix bcU, PartitionedBroadcastMatrix bcV )
+ public RDDQuaternaryFunction3( QuaternaryOperator qop, PartitionedBroadcast<MatrixBlock> bcU, PartitionedBroadcast<MatrixBlock> bcV )
throws DMLRuntimeException
{
super(qop, bcU, bcV);
@@ -490,8 +490,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
MatrixBlock blkOut = new MatrixBlock();
- MatrixBlock mbU = (_pmU!=null)?_pmU.getMatrixBlock((int)ixIn.getRowIndex(), 1) : blkIn2;
- MatrixBlock mbV = (_pmV!=null)?_pmV.getMatrixBlock((int)ixIn.getColumnIndex(), 1) :
+ MatrixBlock mbU = (_pmU!=null)?_pmU.getBlock((int)ixIn.getRowIndex(), 1) : blkIn2;
+ MatrixBlock mbV = (_pmV!=null)?_pmV.getBlock((int)ixIn.getColumnIndex(), 1) :
(_pmU!=null)? blkIn2 : blkIn3;
MatrixBlock mbW = (_qop.hasFourInputs())? blkIn3 : null;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
index e2cb782..8974a7e 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
@@ -42,7 +42,7 @@ import org.apache.sysml.runtime.functionobjects.ReduceRow;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
-import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcast;
import org.apache.sysml.runtime.instructions.spark.functions.AggregateDropCorrectionFunction;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -158,7 +158,7 @@ public class UaggOuterChainSPInstruction extends BinarySPInstruction
}
else
{
- PartitionedBroadcastMatrix bv = sec.getBroadcastForVariable( bcastVar );
+ PartitionedBroadcast<MatrixBlock> bv = sec.getBroadcastForVariable( bcastVar );
//partitioning-preserving map-to-pair (under constraints)
out = in1.mapPartitionsToPair( new RDDMapGenUAggOuterChainFunction(bv, _uaggOp, _aggOp, _bOp, mcIn), noKeyChange );
@@ -314,7 +314,7 @@ public class UaggOuterChainSPInstruction extends BinarySPInstruction
{
private static final long serialVersionUID = 8197406787010296291L;
- private PartitionedBroadcastMatrix _pbc = null;
+ private PartitionedBroadcast<MatrixBlock> _pbc = null;
// Operators
private AggregateUnaryOperator _uaggOp = null;
@@ -327,7 +327,7 @@ public class UaggOuterChainSPInstruction extends BinarySPInstruction
private MatrixValue _tmpVal1 = null;
private MatrixValue _tmpVal2 = null;
- public RDDMapGenUAggOuterChainFunction(PartitionedBroadcastMatrix binput, AggregateUnaryOperator uaggOp, AggregateOperator aggOp, BinaryOperator bOp,
+ public RDDMapGenUAggOuterChainFunction(PartitionedBroadcast<MatrixBlock> binput, AggregateUnaryOperator uaggOp, AggregateOperator aggOp, BinaryOperator bOp,
MatrixCharacteristics mc)
{
_pbc = binput;
@@ -374,7 +374,7 @@ public class UaggOuterChainSPInstruction extends BinarySPInstruction
for(int bidx=1; bidx <= in2_colBlocks; bidx++)
{
- MatrixValue in2Val = _pbc.getMatrixBlock(1, bidx);
+ MatrixValue in2Val = _pbc.getBlock(1, bidx);
//outer block operation
OperationsOnMatrixValues.performBinaryIgnoreIndexes(in1Val, in2Val, _tmpVal1, _bOp);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
index f42c29e..0cfec66 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
@@ -146,7 +146,7 @@ public class WriteSPInstruction extends SPInstruction
//prepare output info according to meta data
String outFmt = input3.getName();
OutputInfo oi = OutputInfo.stringToOutputInfo(outFmt);
-
+
//core matrix/frame write
if( input1.getDataType()==DataType.MATRIX )
processMatrixWriteInstruction(sec, fname, oi);
@@ -311,7 +311,7 @@ public class WriteSPInstruction extends SPInstruction
}
// write meta data file
- MapReduceTool.writeMetaDataFile(fname + ".mtd", input1.getValueType(), DataType.FRAME, mc, oi, formatProperties);
+ MapReduceTool.writeMetaDataFile(fname + ".mtd", input1.getValueType(), null, DataType.FRAME, mc, oi, formatProperties);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
index 13c68e2..6091edf 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
@@ -22,15 +22,16 @@ package org.apache.sysml.runtime.instructions.spark.data;
import java.lang.ref.SoftReference;
import org.apache.spark.broadcast.Broadcast;
+import org.apache.sysml.runtime.controlprogram.caching.CacheBlock;
-public class BroadcastObject extends LineageObject
+public class BroadcastObject<T extends CacheBlock> extends LineageObject
{
//soft reference storage for graceful cleanup in case of memory pressure
- private SoftReference<PartitionedBroadcastMatrix> _bcHandle = null;
+ protected SoftReference<PartitionedBroadcast<T>> _bcHandle = null;
- public BroadcastObject( PartitionedBroadcastMatrix bvar, String varName )
+ public BroadcastObject( PartitionedBroadcast<T> bvar, String varName )
{
- _bcHandle = new SoftReference<PartitionedBroadcastMatrix>(bvar);
+ _bcHandle = new SoftReference<PartitionedBroadcast<T>>(bvar);
_varName = varName;
}
@@ -38,7 +39,8 @@ public class BroadcastObject extends LineageObject
*
* @return
*/
- public PartitionedBroadcastMatrix getBroadcast()
+ @SuppressWarnings("rawtypes")
+ public PartitionedBroadcast getBroadcast()
{
return _bcHandle.get();
}
@@ -50,13 +52,13 @@ public class BroadcastObject extends LineageObject
public boolean isValid()
{
//check for evicted soft reference
- PartitionedBroadcastMatrix pbm = _bcHandle.get();
+ PartitionedBroadcast<T> pbm = _bcHandle.get();
if( pbm == null )
return false;
//check for validity of individual broadcasts
- Broadcast<PartitionedMatrixBlock>[] tmp = pbm.getBroadcasts();
- for( Broadcast<PartitionedMatrixBlock> bc : tmp )
+ Broadcast<PartitionedBlock<T>>[] tmp = pbm.getBroadcasts();
+ for( Broadcast<PartitionedBlock<T>> bc : tmp )
if( !bc.isValid() )
return false;
return true;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java
new file mode 100644
index 0000000..20fcd0b
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java
@@ -0,0 +1,399 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.spark.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
+import java.lang.reflect.Array;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheBlock;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.util.FastBufferedDataInputStream;
+import org.apache.sysml.runtime.util.FastBufferedDataOutputStream;
+import org.apache.sysml.runtime.util.IndexRange;
+
+/**
+ * This class is for partitioned matrix/frame blocks, to be used
+ * as broadcasts. Distributed tasks require block-partitioned broadcasts but a lazy partitioning per
+ * task would create instance-local copies and hence replicate broadcast variables which are shared
+ * by all tasks within an executor.
+ *
+ */
+public class PartitionedBlock<T extends CacheBlock> implements Externalizable
+{
+
+ protected T[] _partBlocks = null;
+ protected long _rlen = -1;
+ protected long _clen = -1;
+ protected int _brlen = -1;
+ protected int _bclen = -1;
+ protected int _offset = 0;
+
+ public PartitionedBlock() {
+ //do nothing (required for Externalizable)
+ }
+
+
+ public long getNumRows() {
+ return _rlen;
+ }
+
+ public long getNumCols() {
+ return _clen;
+ }
+
+ public long getNumRowsPerBlock() {
+ return _brlen;
+ }
+
+ public long getNumColumnsPerBlock() {
+ return _bclen;
+ }
+
+ /**
+ *
+ * @return
+ */
+ public int getNumRowBlocks()
+ {
+ return (int)Math.ceil((double)_rlen/_brlen);
+ }
+
+ /**
+ *
+ * @return
+ */
+ public int getNumColumnBlocks()
+ {
+ return (int)Math.ceil((double)_clen/_bclen);
+ }
+
+ @SuppressWarnings("unchecked")
+ public PartitionedBlock(T block, int brlen, int bclen)
+ {
+ //get the input frame block
+ int rlen = block.getNumRows();
+ int clen = block.getNumColumns();
+
+ //partitioning input broadcast
+ _rlen = rlen;
+ _clen = clen;
+ _brlen = brlen;
+ _bclen = bclen;
+
+ int nrblks = getNumRowBlocks();
+ int ncblks = getNumColumnBlocks();
+
+ try
+ {
+ _partBlocks = (T[])Array.newInstance((block.getClass()), nrblks * ncblks);
+ for( int i=0, ix=0; i<nrblks; i++ )
+ for( int j=0; j<ncblks; j++, ix++ )
+ {
+ T tmp = (T) block.getClass().newInstance();
+ block.sliceOperations(i*_brlen, Math.min((i+1)*_brlen, rlen)-1,
+ j*_bclen, Math.min((j+1)*_bclen, clen)-1, tmp);
+ _partBlocks[ix] = tmp;
+ }
+ }
+ catch(Exception ex) {
+ throw new RuntimeException("Failed partitioning of broadcast variable input.", ex);
+ }
+
+ _offset = 0;
+ }
+
+ @SuppressWarnings("unchecked")
+ public PartitionedBlock(int rlen, int clen, int brlen, int bclen, T block)
+ {
+ //partitioning input broadcast
+ _rlen = rlen;
+ _clen = clen;
+ _brlen = brlen;
+ _bclen = bclen;
+
+ int nrblks = getNumRowBlocks();
+ int ncblks = getNumColumnBlocks();
+ _partBlocks = (T[])Array.newInstance((block.getClass()), nrblks * ncblks);
+
+ }
+
+ /**
+ *
+ * @param rowIndex
+ * @param colIndex
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public T getBlock(int rowIndex, int colIndex)
+ throws DMLRuntimeException
+ {
+ //check for valid block index
+ int nrblks = getNumRowBlocks();
+ int ncblks = getNumColumnBlocks();
+ if( rowIndex <= 0 || rowIndex > nrblks || colIndex <= 0 || colIndex > ncblks ) {
+ throw new DMLRuntimeException("Block indexes ["+rowIndex+","+colIndex+"] out of range ["+nrblks+","+ncblks+"]");
+ }
+
+ //get the requested frame/matrix block
+ int rix = rowIndex - 1;
+ int cix = colIndex - 1;
+ int ix = rix*ncblks+cix - _offset;
+ return _partBlocks[ix];
+ }
+
+ /**
+ *
+ * @param rowIndex
+ * @param colIndex
+ * @param mb
+ * @throws DMLRuntimeException
+ */
+ public void setBlock(int rowIndex, int colIndex, T block)
+ throws DMLRuntimeException
+ {
+ //check for valid block index
+ int nrblks = getNumRowBlocks();
+ int ncblks = getNumColumnBlocks();
+ if( rowIndex <= 0 || rowIndex > nrblks || colIndex <= 0 || colIndex > ncblks ) {
+ throw new DMLRuntimeException("Block indexes ["+rowIndex+","+colIndex+"] out of range ["+nrblks+","+ncblks+"]");
+ }
+
+ //get the requested matrix block
+ int rix = rowIndex - 1;
+ int cix = colIndex - 1;
+ int ix = rix*ncblks+cix - _offset;
+ _partBlocks[ ix ] = block;
+
+ }
+
+ /**
+ *
+ * @param offset
+ * @param numBlks
+ * @return
+ */
+ @SuppressWarnings("unchecked")
+ public PartitionedBlock<T> createPartition( int offset, int numBlks, T block )
+ {
+ PartitionedBlock<T> ret = new PartitionedBlock<T>();
+ ret._rlen = _rlen;
+ ret._clen = _clen;
+ ret._brlen = _brlen;
+ ret._bclen = _bclen;
+
+ _partBlocks = (T[])Array.newInstance(block.getClass(), numBlks);
+ ret._offset = offset;
+ System.arraycopy(_partBlocks, offset, ret._partBlocks, 0, numBlks);
+
+ return ret;
+ }
+
+ /**
+ *
+ * @return
+ */
+ public long getInMemorySize()
+ {
+ long ret = 24; //header
+ ret += 32; //block array
+
+ if( _partBlocks != null )
+ for( T block : _partBlocks )
+ ret += block.getInMemorySize();
+
+ return ret;
+ }
+
+ /**
+ *
+ * @return
+ */
+
+ public long getExactSerializedSize()
+ {
+ long ret = 24; //header
+
+ if( _partBlocks != null )
+ for( T block : _partBlocks )
+ ret += block.getExactSerializedSize();
+
+ return ret;
+ }
+
+ /**
+ * Utility for slice operations over partitioned matrices, where the index range can cover
+ * multiple blocks. The result is always a single result matrix block. All semantics are
+ * equivalent to the core matrix block slice operations.
+ *
+ * @param rl
+ * @param ru
+ * @param cl
+ * @param cu
+ * @param matrixBlock
+ * @return
+ * @throws DMLRuntimeException
+ */
+ @SuppressWarnings("unchecked")
+ public T sliceOperations(long rl, long ru, long cl, long cu, T block)
+ throws DMLRuntimeException
+ {
+ int lrl = (int) rl;
+ int lru = (int) ru;
+ int lcl = (int) cl;
+ int lcu = (int) cu;
+
+ ArrayList<Pair<?, ?>> allBlks = block.getPairList();
+ int start_iix = (lrl-1)/_brlen+1;
+ int end_iix = (lru-1)/_brlen+1;
+ int start_jix = (lcl-1)/_bclen+1;
+ int end_jix = (lcu-1)/_bclen+1;
+
+ for( int iix = start_iix; iix <= end_iix; iix++ )
+ for(int jix = start_jix; jix <= end_jix; jix++)
+ {
+ T in = getBlock(iix, jix);
+ IndexRange ixrange = new IndexRange(rl, ru, cl, cu);
+ ArrayList<Pair<?, ?>> outlist = block.performSlice(ixrange, _brlen, _bclen, iix, jix, in);
+ allBlks.addAll(outlist);
+ }
+
+ if(allBlks.size() == 1) {
+ return (T) allBlks.get(0).getValue();
+ }
+ else {
+ //allocate output matrix
+ Constructor<?> constr;
+ try {
+ constr = block.getClass().getConstructor(int.class, int.class, boolean.class);
+ T ret = (T) constr.newInstance(lru-lrl+1, lcu-lcl+1, false);
+ for(Pair<?, ?> kv : allBlks) {
+ ret.merge((T)kv.getValue(), false);
+ }
+ return ret;
+ } catch (Exception e) {
+ throw new DMLRuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Redirects the default java serialization via externalizable to our default
+ * hadoop writable serialization for efficient broadcast deserialization.
+ *
+ * @param is
+ * @throws IOException
+ */
+ public void readExternal(ObjectInput is)
+ throws IOException
+ {
+ DataInput dis = is;
+
+ if( is instanceof ObjectInputStream ) {
+ //fast deserialize of dense/sparse blocks
+ ObjectInputStream ois = (ObjectInputStream)is;
+ dis = new FastBufferedDataInputStream(ois);
+ }
+
+ readHeaderAndPayload(dis);
+ }
+
+ /**
+ * Redirects the default java serialization via externalizable to our default
+ * hadoop writable serialization for efficient broadcast serialization.
+ *
+ * @param is
+ * @throws IOException
+ */
+ public void writeExternal(ObjectOutput os)
+ throws IOException
+ {
+ if( os instanceof ObjectOutputStream ) {
+ //fast serialize of dense/sparse blocks
+ ObjectOutputStream oos = (ObjectOutputStream)os;
+ FastBufferedDataOutputStream fos = new FastBufferedDataOutputStream(oos);
+ writeHeaderAndPayload(fos);
+ fos.flush();
+ }
+ else {
+ //default serialize (general case)
+ writeHeaderAndPayload(os);
+ }
+ }
+
+ /**
+ *
+ * @param dos
+ * @throws IOException
+ */
+ private void writeHeaderAndPayload(DataOutput dos)
+ throws IOException
+ {
+ dos.writeLong(_rlen);
+ dos.writeLong(_clen);
+ dos.writeInt(_brlen);
+ dos.writeInt(_bclen);
+ dos.writeInt(_offset);
+ dos.writeInt(_partBlocks.length);
+
+ for( T block : _partBlocks )
+ block.write(dos);
+ }
+
+ /**
+ *
+ * @param din
+ * @throws IOException
+ */
+ @SuppressWarnings("unchecked")
+ private void readHeaderAndPayload(DataInput dis)
+ throws IOException
+ {
+ _rlen = dis.readInt();
+ _clen = dis.readInt();
+ _brlen = dis.readInt();
+ _bclen = dis.readInt();
+ _offset = dis.readInt();
+
+ int len = dis.readInt();
+
+ try
+ {
+ _partBlocks = (T[])Array.newInstance(getClass(), len);
+ for( int i=0; i<len; i++ ) {
+ _partBlocks[i].readFields(dis);
+ }
+ }
+ catch(Exception ex) {
+ throw new RuntimeException("Failed partitioning of broadcast variable input.", ex);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java
new file mode 100644
index 0000000..d924cc0
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.spark.data;
+
+import java.io.Serializable;
+
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheBlock;
+
+/**
+ * This class is a wrapper around an array of broadcasts of partitioned matrix/frame blocks,
+ * which is required due to 2GB limitations of Spark's broadcast handling. Without this
+ * partitioning of Broadcast<PartitionedBlock> into Broadcast<PartitionedBlock>[],
+ * we got java.lang.IllegalArgumentException: Size exceeds Integer.MAX_VALUE issue.
+ * Despite various jiras, this issue still showed up in Spark 1.4/1.5.
+ *
+ */
+public class PartitionedBroadcast<T extends CacheBlock> implements Serializable
+{
+ private static final long serialVersionUID = 7041959166079438401L;
+
+ protected static final long BROADCAST_PARTSIZE = 200L*1024*1024; //200M cells ~ 1.6GB
+
+ private Broadcast<PartitionedBlock<T>>[] _pbc = null;
+
+ public PartitionedBroadcast() {
+ //do nothing (required for Externalizable)
+ }
+
+ public PartitionedBroadcast(Broadcast<PartitionedBlock<T>>[] broadcasts)
+ {
+ _pbc = broadcasts;
+ }
+
+ public Broadcast<PartitionedBlock<T>>[] getBroadcasts() {
+ return _pbc;
+ }
+
+ /**
+ *
+ * @return
+ */
+ public int getNumRowBlocks() {
+ return _pbc[0].value().getNumRowBlocks();
+ }
+
+ public int getNumColumnBlocks() {
+ return _pbc[0].value().getNumColumnBlocks();
+ }
+
+ /**
+ *
+ * @param rlen
+ * @param clen
+ * @param brlen
+ * @param bclen
+ * @return
+ */
+ public static int computeBlocksPerPartition(long rlen, long clen, long brlen, long bclen) {
+ return (int) Math.floor( BROADCAST_PARTSIZE /
+ Math.min(rlen, brlen) / Math.min(clen, bclen));
+ }
+ /**
+ *
+ * @param rowIndex
+ * @param colIndex
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public T getBlock(int rowIndex, int colIndex)
+ throws DMLRuntimeException
+ {
+ int pix = 0;
+
+ if( _pbc.length > 1 ) {
+ //compute partition index
+ PartitionedBlock<T> tmp = _pbc[0].value();
+ int numPerPart = computeBlocksPerPartition(tmp.getNumRows(), tmp.getNumCols(),
+ tmp.getNumRowsPerBlock(), tmp.getNumColumnsPerBlock());
+ int ix = (rowIndex-1)*tmp.getNumColumnBlocks()+(colIndex-1);
+ pix = ix / numPerPart;
+ }
+
+ return _pbc[pix].value().getBlock(rowIndex, colIndex);
+ }
+
+ public T sliceOperations(long rl, long ru, long cl, long cu, T block)
+ throws DMLRuntimeException
+ {
+ T ret = null;
+
+ for( Broadcast<PartitionedBlock<T>> bc : _pbc ) {
+ PartitionedBlock<T> pm = bc.value();
+ T tmp = pm.sliceOperations(rl, ru, cl, cu, block);
+ if( ret != null )
+ ret.merge(tmp, false);
+ else
+ ret = tmp;
+ }
+
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
deleted file mode 100644
index f79d8c1..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.spark.data;
-
-import java.io.Serializable;
-
-import org.apache.spark.broadcast.Broadcast;
-
-import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-
-/**
- * This class is a wrapper around an array of broadcasts of partitioned matrix blocks,
- * which is required due to 2GB limitations of Spark's broadcast handling. Without this
- * partitioning of Broadcast<PartitionedMatrixBlock> into Broadcast<PartitionedMatrixBlock>[],
- * we got java.lang.IllegalArgumentException: Size exceeds Integer.MAX_VALUE issue.
- * Despite various jiras, this issue still showed up in Spark 1.4/1.5.
- *
- */
-public class PartitionedBroadcastMatrix implements Serializable
-{
- private static final long serialVersionUID = 1225135967889810877L;
- private static final long BROADCAST_PARTSIZE = 200L*1024*1024; //200M cells ~ 1.6GB
-
- private Broadcast<PartitionedMatrixBlock>[] _pbc = null;
-
- public PartitionedBroadcastMatrix(Broadcast<PartitionedMatrixBlock>[] broadcasts)
- {
- _pbc = broadcasts;
- }
-
- public Broadcast<PartitionedMatrixBlock>[] getBroadcasts() {
- return _pbc;
- }
-
- /**
- *
- * @return
- */
- public int getNumRowBlocks() {
- return _pbc[0].value().getNumRowBlocks();
- }
-
- public int getNumColumnBlocks() {
- return _pbc[0].value().getNumColumnBlocks();
- }
-
- /**
- *
- * @param rowIndex
- * @param colIndex
- * @return
- * @throws DMLRuntimeException
- */
- public MatrixBlock getMatrixBlock(int rowIndex, int colIndex)
- throws DMLRuntimeException
- {
- if( _pbc.length > 1 ) {
- //compute partition index
- PartitionedMatrixBlock tmp = _pbc[0].value();
- int numPerPart = computeBlocksPerPartition(tmp.getNumRows(), tmp.getNumCols(),
- tmp.getNumRowsPerBlock(), tmp.getNumColumnsPerBlock());
- int ix = (rowIndex-1)*tmp.getNumColumnBlocks()+(colIndex-1);
- int pix = ix / numPerPart;
-
- //get matrix block from partition
- return _pbc[pix].value().getMatrixBlock(rowIndex, colIndex);
- }
- else { //single partition
- return _pbc[0].value().getMatrixBlock(rowIndex, colIndex);
- }
-
- }
-
- public MatrixBlock sliceOperations(long rl, long ru, long cl, long cu, MatrixBlock matrixBlock)
- throws DMLRuntimeException
- {
- MatrixBlock ret = null;
-
- for( Broadcast<PartitionedMatrixBlock> bc : _pbc ) {
- PartitionedMatrixBlock pm = bc.value();
- MatrixBlock tmp = pm.sliceOperations(rl, ru, cl, cu, new MatrixBlock());
- if( ret != null )
- ret.merge(tmp, false);
- else
- ret = tmp;
- }
-
- return ret;
- }
-
- /**
- *
- * @param rlen
- * @param clen
- * @param brlen
- * @param bclen
- * @return
- */
- public static int computeBlocksPerPartition(long rlen, long clen, long brlen, long bclen) {
- return (int) Math.floor( BROADCAST_PARTSIZE /
- Math.min(rlen, brlen) / Math.min(clen, bclen));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedMatrixBlock.java
deleted file mode 100644
index 901b130..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedMatrixBlock.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.spark.data;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutput;
-import java.io.ObjectOutputStream;
-import java.util.ArrayList;
-
-import scala.Tuple2;
-
-import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
-import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
-import org.apache.sysml.runtime.util.FastBufferedDataInputStream;
-import org.apache.sysml.runtime.util.FastBufferedDataOutputStream;
-import org.apache.sysml.runtime.util.IndexRange;
-
-/**
- * The main purpose of this class is to provide a handle for partitioned matrix blocks, to be used
- * as broadcasts. Distributed tasks require block-partitioned broadcasts but a lazy partitioning per
- * task would create instance-local copies and hence replicate broadcast variables which are shared
- * by all tasks within an executor.
- *
- */
-public class PartitionedMatrixBlock implements Externalizable
-{
-
- private static final long serialVersionUID = -5706923809800365593L;
-
- private MatrixBlock[] _partBlocks = null;
- private int _rlen = -1;
- private int _clen = -1;
- private int _brlen = -1;
- private int _bclen = -1;
- private int _offset = 0;
-
- public PartitionedMatrixBlock() {
- //do nothing (required for Externalizable)
- }
-
- public PartitionedMatrixBlock(MatrixBlock mb, int brlen, int bclen)
- {
- //get the input matrix block
- int rlen = mb.getNumRows();
- int clen = mb.getNumColumns();
-
- //partitioning input broadcast
- _rlen = rlen;
- _clen = clen;
- _brlen = brlen;
- _bclen = bclen;
-
- int nrblks = getNumRowBlocks();
- int ncblks = getNumColumnBlocks();
- _partBlocks = new MatrixBlock[nrblks * ncblks];
-
- try
- {
- for( int i=0, ix=0; i<nrblks; i++ )
- for( int j=0; j<ncblks; j++, ix++ )
- {
- MatrixBlock tmp = new MatrixBlock();
- mb.sliceOperations(i*brlen, Math.min((i+1)*brlen, rlen)-1,
- j*bclen, Math.min((j+1)*bclen, clen)-1, tmp);
- _partBlocks[ix] = tmp;
- }
- }
- catch(Exception ex) {
- throw new RuntimeException("Failed partitioning of broadcast variable input.", ex);
- }
-
- _offset = 0;
- }
-
- public PartitionedMatrixBlock(int rlen, int clen, int brlen, int bclen)
- {
- //partitioning input broadcast
- _rlen = rlen;
- _clen = clen;
- _brlen = brlen;
- _bclen = bclen;
-
- int nrblks = getNumRowBlocks();
- int ncblks = getNumColumnBlocks();
- _partBlocks = new MatrixBlock[nrblks * ncblks];
- }
-
- public long getNumRows() {
- return _rlen;
- }
-
- public long getNumCols() {
- return _clen;
- }
-
- public long getNumRowsPerBlock() {
- return _brlen;
- }
-
- public long getNumColumnsPerBlock() {
- return _bclen;
- }
-
- /**
- *
- * @return
- */
- public int getNumRowBlocks()
- {
- return (int)Math.ceil((double)_rlen/_brlen);
- }
-
- /**
- *
- * @return
- */
- public int getNumColumnBlocks()
- {
- return (int)Math.ceil((double)_clen/_bclen);
- }
-
- /**
- *
- * @param rowIndex
- * @param colIndex
- * @return
- * @throws DMLRuntimeException
- */
- public MatrixBlock getMatrixBlock(int rowIndex, int colIndex)
- throws DMLRuntimeException
- {
- //check for valid block index
- int nrblks = getNumRowBlocks();
- int ncblks = getNumColumnBlocks();
- if( rowIndex <= 0 || rowIndex > nrblks || colIndex <= 0 || colIndex > ncblks ) {
- throw new DMLRuntimeException("Block indexes ["+rowIndex+","+colIndex+"] out of range ["+nrblks+","+ncblks+"]");
- }
-
- //get the requested matrix block
- int rix = rowIndex - 1;
- int cix = colIndex - 1;
- int ix = rix*ncblks+cix - _offset;
- return _partBlocks[ ix ];
- }
-
- /**
- *
- * @param rowIndex
- * @param colIndex
- * @param mb
- * @throws DMLRuntimeException
- */
- public void setMatrixBlock(int rowIndex, int colIndex, MatrixBlock mb)
- throws DMLRuntimeException
- {
- //check for valid block index
- int nrblks = getNumRowBlocks();
- int ncblks = getNumColumnBlocks();
- if( rowIndex <= 0 || rowIndex > nrblks || colIndex <= 0 || colIndex > ncblks ) {
- throw new DMLRuntimeException("Block indexes ["+rowIndex+","+colIndex+"] out of range ["+nrblks+","+ncblks+"]");
- }
-
- //get the requested matrix block
- int rix = rowIndex - 1;
- int cix = colIndex - 1;
- int ix = rix*ncblks+cix - _offset;
- _partBlocks[ ix ] = mb;
-
- }
-
- /**
- *
- * @return
- */
- public long estimateSizeInMemory()
- {
- long ret = 24; //header
- ret += 32; //block array
-
- if( _partBlocks != null )
- for( MatrixBlock mb : _partBlocks )
- ret += mb.estimateSizeInMemory();
-
- return ret;
- }
-
- /**
- *
- * @return
- */
- public long estimateSizeOnDisk()
- {
- long ret = 24; //header
-
- if( _partBlocks != null )
- for( MatrixBlock mb : _partBlocks )
- ret += mb.estimateSizeOnDisk();
-
- return ret;
- }
-
- /**
- *
- * @param offset
- * @param numBlks
- * @return
- */
- public PartitionedMatrixBlock createPartition( int offset, int numBlks )
- {
- PartitionedMatrixBlock ret = new PartitionedMatrixBlock();
- ret._rlen = _rlen;
- ret._clen = _clen;
- ret._brlen = _brlen;
- ret._bclen = _bclen;
- ret._partBlocks = new MatrixBlock[numBlks];
- ret._offset = offset;
-
- System.arraycopy(_partBlocks, offset, ret._partBlocks, 0, numBlks);
-
- return ret;
- }
-
- /**
- * Utility for slice operations over partitioned matrices, where the index range can cover
- * multiple blocks. The result is always a single result matrix block. All semantics are
- * equivalent to the core matrix block slice operations.
- *
- * @param rl
- * @param ru
- * @param cl
- * @param cu
- * @param matrixBlock
- * @return
- * @throws DMLRuntimeException
- */
- public MatrixBlock sliceOperations(long rl, long ru, long cl, long cu, MatrixBlock matrixBlock)
- throws DMLRuntimeException
- {
- int lrl = (int) rl;
- int lru = (int) ru;
- int lcl = (int) cl;
- int lcu = (int) cu;
-
- ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> allBlks = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>();
- int start_iix = (lrl-1)/_brlen+1;
- int end_iix = (lru-1)/_brlen+1;
- int start_jix = (lcl-1)/_bclen+1;
- int end_jix = (lcu-1)/_bclen+1;
-
- for( int iix = start_iix; iix <= end_iix; iix++ )
- for(int jix = start_jix; jix <= end_jix; jix++)
- {
- MatrixBlock in = getMatrixBlock(iix, jix);
- IndexedMatrixValue imv = new IndexedMatrixValue(new MatrixIndexes(iix, jix), in);
-
- ArrayList<IndexedMatrixValue> outlist = new ArrayList<IndexedMatrixValue>();
- IndexRange ixrange = new IndexRange(rl, ru, cl, cu);
- OperationsOnMatrixValues.performSlice(imv, ixrange, _brlen, _bclen, outlist);
- allBlks.addAll(SparkUtils.fromIndexedMatrixBlock(outlist));
- }
-
- if(allBlks.size() == 1) {
- return allBlks.get(0)._2;
- }
- else {
- //allocate output matrix
- MatrixBlock ret = new MatrixBlock(lru-lrl+1, lcu-lcl+1, false);
- for(Tuple2<MatrixIndexes, MatrixBlock> kv : allBlks) {
- ret.merge(kv._2, false);
- }
- return ret;
- }
- }
-
- /**
- * Redirects the default java serialization via externalizable to our default
- * hadoop writable serialization for efficient broadcast deserialization.
- *
- * @param is
- * @throws IOException
- */
- public void readExternal(ObjectInput is)
- throws IOException
- {
- DataInput dis = is;
-
- if( is instanceof ObjectInputStream ) {
- //fast deserialize of dense/sparse blocks
- ObjectInputStream ois = (ObjectInputStream)is;
- dis = new FastBufferedDataInputStream(ois);
- }
-
- readHeaderAndPayload(dis);
- }
-
- /**
- * Redirects the default java serialization via externalizable to our default
- * hadoop writable serialization for efficient broadcast serialization.
- *
- * @param is
- * @throws IOException
- */
- public void writeExternal(ObjectOutput os)
- throws IOException
- {
- if( os instanceof ObjectOutputStream ) {
- //fast serialize of dense/sparse blocks
- ObjectOutputStream oos = (ObjectOutputStream)os;
- FastBufferedDataOutputStream fos = new FastBufferedDataOutputStream(oos);
- writeHeaderAndPayload(fos);
- fos.flush();
- }
- else {
- //default serialize (general case)
- writeHeaderAndPayload(os);
- }
- }
-
- /**
- *
- * @param dos
- * @throws IOException
- */
- private void writeHeaderAndPayload(DataOutput dos)
- throws IOException
- {
- dos.writeInt(_rlen);
- dos.writeInt(_clen);
- dos.writeInt(_brlen);
- dos.writeInt(_bclen);
- dos.writeInt(_offset);
- dos.writeInt(_partBlocks.length);
- for( MatrixBlock mb : _partBlocks )
- mb.write(dos);
- }
-
- /**
- *
- * @param din
- * @throws IOException
- */
- private void readHeaderAndPayload(DataInput dis)
- throws IOException
- {
- _rlen = dis.readInt();
- _clen = dis.readInt();
- _brlen = dis.readInt();
- _bclen = dis.readInt();
- _offset = dis.readInt();
-
- int len = dis.readInt();
- _partBlocks = new MatrixBlock[len];
-
- for( int i=0; i<len; i++ ) {
- _partBlocks[i] = new MatrixBlock();
- _partBlocks[i].readFields(dis);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java
index 9e31878..df891f7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java
@@ -57,4 +57,4 @@ public class CopyFrameBlockPairFunction implements PairFunction<Tuple2<LongWrita
return new Tuple2<Long,FrameBlock>(arg0._1().get(), arg0._2());
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java
index e8c6dbe..2a3aedf 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java
@@ -27,7 +27,7 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
import org.apache.sysml.hops.OptimizerUtils;
-import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcast;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.WeightedCell;
@@ -139,9 +139,9 @@ public abstract class ExtractGroup implements Serializable
{
private static final long serialVersionUID = 5709955602290131093L;
- private PartitionedBroadcastMatrix _pbm = null;
+ private PartitionedBroadcast<MatrixBlock> _pbm = null;
- public ExtractGroupBroadcast( PartitionedBroadcastMatrix pbm, long bclen, long ngroups, Operator op ) {
+ public ExtractGroupBroadcast( PartitionedBroadcast<MatrixBlock> pbm, long bclen, long ngroups, Operator op ) {
super(bclen, ngroups, op);
_pbm = pbm;
}
@@ -152,7 +152,7 @@ public abstract class ExtractGroup implements Serializable
throws Exception
{
MatrixIndexes ix = arg._1;
- MatrixBlock group = _pbm.getMatrixBlock((int)ix.getRowIndex(), 1);
+ MatrixBlock group = _pbm.getBlock((int)ix.getRowIndex(), 1);
MatrixBlock target = arg._2;
return execute(ix, group, target);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
index b110861..bc6035d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpFunction.java
@@ -25,7 +25,7 @@ import org.apache.spark.broadcast.Broadcast;
import scala.Tuple2;
import org.apache.sysml.lops.BinaryM.VectorType;
-import org.apache.sysml.runtime.instructions.spark.data.PartitionedMatrixBlock;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
@@ -36,10 +36,10 @@ public class MatrixVectorBinaryOpFunction implements PairFunction<Tuple2<MatrixI
private static final long serialVersionUID = -7695883019452417300L;
private BinaryOperator _op = null;
- private Broadcast<PartitionedMatrixBlock> _pmV = null;
+ private Broadcast<PartitionedBlock<MatrixBlock>> _pmV = null;
private VectorType _vtype = null;
- public MatrixVectorBinaryOpFunction( BinaryOperator op, Broadcast<PartitionedMatrixBlock> binput, VectorType vtype )
+ public MatrixVectorBinaryOpFunction( BinaryOperator op, Broadcast<PartitionedBlock<MatrixBlock>> binput, VectorType vtype )
{
_op = op;
_pmV = binput;
@@ -56,7 +56,7 @@ public class MatrixVectorBinaryOpFunction implements PairFunction<Tuple2<MatrixI
//get the rhs block
int rix= (int)((_vtype==VectorType.COL_VECTOR) ? ix.getRowIndex() : 1);
int cix= (int)((_vtype==VectorType.COL_VECTOR) ? 1 : ix.getColumnIndex());
- MatrixBlock in2 = _pmV.value().getMatrixBlock(rix, cix);
+ MatrixBlock in2 = _pmV.value().getBlock(rix, cix);
//execute the binary operation
MatrixBlock ret = (MatrixBlock) (in1.binaryOperations (_op, in2, new MatrixBlock()));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
index b298693..597c028 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
@@ -27,7 +27,7 @@ import scala.Tuple2;
import org.apache.sysml.lops.BinaryM.VectorType;
import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
-import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcast;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
@@ -37,10 +37,10 @@ public class MatrixVectorBinaryOpPartitionFunction implements PairFlatMapFunctio
private static final long serialVersionUID = 9096091404578628534L;
private BinaryOperator _op = null;
- private PartitionedBroadcastMatrix _pmV = null;
+ private PartitionedBroadcast<MatrixBlock> _pmV = null;
private VectorType _vtype = null;
- public MatrixVectorBinaryOpPartitionFunction( BinaryOperator op, PartitionedBroadcastMatrix binput, VectorType vtype )
+ public MatrixVectorBinaryOpPartitionFunction( BinaryOperator op, PartitionedBroadcast<MatrixBlock> binput, VectorType vtype )
{
_op = op;
_pmV = binput;
@@ -76,7 +76,7 @@ public class MatrixVectorBinaryOpPartitionFunction implements PairFlatMapFunctio
//get the rhs block
int rix= (int)((_vtype==VectorType.COL_VECTOR) ? ix.getRowIndex() : 1);
int cix= (int)((_vtype==VectorType.COL_VECTOR) ? 1 : ix.getColumnIndex());
- MatrixBlock in2 = _pmV.getMatrixBlock(rix, cix);
+ MatrixBlock in2 = _pmV.getBlock(rix, cix);
//execute the binary operation
MatrixBlock ret = (MatrixBlock) (in1.binaryOperations (_op, in2, new MatrixBlock()));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
index 2947c72..3b90ff3 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
@@ -25,7 +25,7 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
-import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcastMatrix;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcast;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
@@ -35,9 +35,9 @@ public class OuterVectorBinaryOpFunction implements PairFlatMapFunction<Tuple2<M
private static final long serialVersionUID = 1730704346934726826L;
private BinaryOperator _op;
- private PartitionedBroadcastMatrix _pmV;
+ private PartitionedBroadcast<MatrixBlock> _pmV;
- public OuterVectorBinaryOpFunction( BinaryOperator op, PartitionedBroadcastMatrix binput )
+ public OuterVectorBinaryOpFunction( BinaryOperator op, PartitionedBroadcast<MatrixBlock> binput )
{
_op = op;
_pmV = binput;
@@ -84,7 +84,7 @@ public class OuterVectorBinaryOpFunction implements PairFlatMapFunction<Tuple2<M
MatrixIndexes ix = _currBlk._1();
MatrixBlock in1 = _currBlk._2();
- MatrixBlock in2 = _pmV.getMatrixBlock(1, _currPos);
+ MatrixBlock in2 = _pmV.getBlock(1, _currPos);
MatrixBlock resultBlk = (MatrixBlock)in1.binaryOperations (_op, in2, new MatrixBlock());
resultBlk.examSparsity();
ret = new Tuple2<MatrixIndexes,MatrixBlock>(
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
index 274efe0..b63724a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
@@ -178,7 +178,6 @@ public class FrameRDDConverterUtils
return textCellToBinaryBlockLongIndex(sc, input, mcOut, lschema);
}
-
/**
*
* @param sc
@@ -198,7 +197,7 @@ public class FrameRDDConverterUtils
//aggregate partial matrix blocks
JavaPairRDD<Long,FrameBlock> out =
- RDDAggregateUtils.mergeByFrameKey( output );
+ (JavaPairRDD<Long, FrameBlock>) RDDAggregateUtils.mergeByFrameKey( output );
return out;
}
@@ -264,7 +263,7 @@ public class FrameRDDConverterUtils
//aggregate partial frame blocks
if(mcIn.getCols() > mcIn.getColsPerBlock())
- out = RDDAggregateUtils.mergeByFrameKey( out ); //TODO: Will need better merger
+ out = (JavaPairRDD<Long, FrameBlock>) RDDAggregateUtils.mergeByFrameKey( out );
}
else
out = input.mapToPair(new MatrixToBinaryBlockOneColumnBlockFunction(mcIn));
@@ -396,8 +395,6 @@ public class FrameRDDConverterUtils
private boolean _fill = false;
private int _maxRowsPerBlock = -1;
- protected static final int BUFFER_SIZE = 1 * 1000 * 1000; //1M elements, size of default matrix block
-
public CSVToBinaryBlockFunction(MatrixCharacteristics mc, boolean hasHeader, String delim, boolean fill)
{
@@ -405,7 +402,7 @@ public class FrameRDDConverterUtils
_hasHeader = hasHeader;
_delim = delim;
_fill = fill;
- _maxRowsPerBlock = Math.max((int) (BUFFER_SIZE/_clen), 1);
+ _maxRowsPerBlock = Math.max((int) (FrameBlock.BUFFER_SIZE/_clen), 1);
}
@Override
@@ -529,10 +526,7 @@ public class FrameRDDConverterUtils
{
private static final long serialVersionUID = -729614449626680946L;
- //internal buffer size (aligned w/ default matrix block size)
- protected static final int BUFFER_SIZE = 1 * 1000 * 1000; //1M elements (8MB), size of default matrix block
protected int _bufflen = -1;
-
protected long _rlen = -1;
protected long _clen = -1;
@@ -542,7 +536,7 @@ public class FrameRDDConverterUtils
_clen = mc.getCols();
//determine upper bounded buffer len
- _bufflen = (int) Math.min(_rlen*_clen, BUFFER_SIZE);
+ _bufflen = (int) Math.min(_rlen*_clen, FrameBlock.BUFFER_SIZE);
}
@@ -624,15 +618,12 @@ public class FrameRDDConverterUtils
private int _maxRowsPerBlock = -1;
- protected static final int BUFFER_SIZE = 1 * 1000 * 1000; //1M elements (Default matrix block size)
-
-
public MatrixToBinaryBlockFunction(MatrixCharacteristics mc)
{
_brlen = mc.getRowsPerBlock();
_bclen = mc.getColsPerBlock();
_clen = mc.getCols();
- _maxRowsPerBlock = Math.max((int) (BUFFER_SIZE/_clen), 1);
+ _maxRowsPerBlock = Math.max((int) (FrameBlock.BUFFER_SIZE/_clen), 1);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
index 48a8496..288a8b5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
@@ -38,7 +38,7 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.functionobjects.SortIndex;
-import org.apache.sysml.runtime.instructions.spark.data.PartitionedMatrixBlock;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBlock;
import org.apache.sysml.runtime.instructions.spark.data.RowMatrixBlock;
import org.apache.sysml.runtime.instructions.spark.functions.ReplicateVectorFunction;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -218,8 +218,8 @@ public class RDDSortUtils
sortedIxSrc.quickSetValue((int)sortedIx.quickGetValue(i,0)-1, 0, i+1);
//broadcast index vector
- PartitionedMatrixBlock pmb = new PartitionedMatrixBlock(sortedIxSrc, brlen, bclen);
- Broadcast<PartitionedMatrixBlock> _pmb = sec.getSparkContext().broadcast(pmb);
+ PartitionedBlock<MatrixBlock> pmb = new PartitionedBlock<MatrixBlock>(sortedIxSrc, brlen, bclen);
+ Broadcast<PartitionedBlock<MatrixBlock>> _pmb = sec.getSparkContext().broadcast(pmb);
//sort data with broadcast index vector
JavaPairRDD<MatrixIndexes, RowMatrixBlock> ret = data
@@ -641,9 +641,9 @@ public class RDDSortUtils
private long _rlen = -1;
private int _brlen = -1;
- private Broadcast<PartitionedMatrixBlock> _pmb = null;
+ private Broadcast<PartitionedBlock<MatrixBlock>> _pmb = null;
- public ShuffleMatrixBlockRowsInMemFunction(long rlen, int brlen, Broadcast<PartitionedMatrixBlock> pmb)
+ public ShuffleMatrixBlockRowsInMemFunction(long rlen, int brlen, Broadcast<PartitionedBlock<MatrixBlock>> pmb)
{
_rlen = rlen;
_brlen = brlen;
@@ -694,7 +694,7 @@ public class RDDSortUtils
//produce next output tuple
MatrixIndexes ixmap = _currBlk._1();
MatrixBlock data = _currBlk._2();
- MatrixBlock mbTargetIndex = _pmb.value().getMatrixBlock((int)ixmap.getRowIndex(), 1);
+ MatrixBlock mbTargetIndex = _pmb.value().getBlock((int)ixmap.getRowIndex(), 1);
long valix = (long) mbTargetIndex.getValue(_currPos, 0);
long rix = UtilFunctions.computeBlockIndex(valix, _brlen);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
index 98975da..90d0148 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
@@ -92,6 +92,29 @@ public class SparkUtils
return ret;
}
+ /**
+ *
+ * @param in
+ * @return
+ */
+ public static Pair<MatrixIndexes,MatrixBlock> fromIndexedMatrixBlockToPair( IndexedMatrixValue in ){
+ return new Pair<MatrixIndexes,MatrixBlock>(in.getIndexes(), (MatrixBlock)in.getValue());
+ }
+
+ /**
+ *
+ * @param in
+ * @return
+ */
+ public static ArrayList<Pair<MatrixIndexes,MatrixBlock>> fromIndexedMatrixBlockToPair( ArrayList<IndexedMatrixValue> in )
+ {
+ ArrayList<Pair<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Pair<MatrixIndexes,MatrixBlock>>();
+ for( IndexedMatrixValue imv : in )
+ ret.add(fromIndexedMatrixBlockToPair(imv));
+
+ return ret;
+ }
+
/**
*
@@ -102,7 +125,6 @@ public class SparkUtils
return new Tuple2<Long, FrameBlock>(in.getKey(), in.getValue());
}
-
/**
*
* @param in
@@ -132,6 +154,25 @@ public class SparkUtils
/**
*
+ * @param in
+ * @return
+ */
+ public static Pair<Long,FrameBlock> toIndexedFrameBlock( Tuple2<Long,FrameBlock> in ) {
+ return new Pair<Long,FrameBlock>(in._1(), in._2());
+ }
+
+ /**
+ *
+ * @param ix
+ * @param mb
+ * @return
+ */
+ public static Pair<Long,FrameBlock> toIndexedFrameBlock( Long ix, FrameBlock fb ) {
+ return new Pair<Long,FrameBlock>(ix, fb);
+ }
+
+ /**
+ *
* @param mb
* @param blen
* @return
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index 0065fe8..e51bd9d 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -46,10 +46,12 @@ import org.apache.sysml.runtime.util.UtilFunctions;
*
*/
@SuppressWarnings({"rawtypes","unchecked"}) //allow generic native arrays
-public class FrameBlock implements Writable, CacheBlock, Externalizable
+public class FrameBlock implements Writable, CacheBlock, Externalizable
{
private static final long serialVersionUID = -3993450030207130665L;
+ public static final int BUFFER_SIZE = 1 * 1000 * 1000; //1M elements, size of default matrix block
+
//internal configuration
private static final boolean REUSE_RECODE_MAPS = true;
@@ -146,6 +148,16 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
}
/**
+ * Sets the schema of the frame block.
+ *
+ * @return
+ */
+ public void setSchema(List<ValueType> schema) {
+ _schema = schema;
+ _colnames = createColNames(schema.size());
+ }
+
+ /**
* Returns the column names of the frame block.
*
* @return
@@ -288,13 +300,14 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @param val
*/
public void set(int r, int c, Object val) {
- _coldata.get(c).set(r, val);
+ _coldata.get(c).set(r, UtilFunctions.objectToObject(_schema.get(c), val));
}
-
+
public void reset(int nrow)
{
getSchema().clear();
getColumnNames().clear();
+
if(_coldata != null)
for(int i=0; i < _coldata.size(); ++i)
_coldata.get(i)._size = nrow;
@@ -567,6 +580,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
throw new DMLRuntimeException("Invalid values for frame indexing: ["+(rl+1)+":"+(ru+1)+"," + (cl+1)+":"+(cu+1)+"] " +
"must be within frame dimensions ["+getNumRows()+","+getNumColumns()+"].");
}
+
if ( (ru-rl+1) < rhsFrame.getNumRows() || (cu-cl+1) < rhsFrame.getNumColumns()) {
throw new DMLRuntimeException("Invalid values for frame indexing: " +
"dimensions of the source frame ["+rhsFrame.getNumRows()+"x" + rhsFrame.getNumColumns() + "] " +
@@ -574,12 +588,14 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
(rl+1) +":" + (ru+1) + ", " + (cl+1) + ":" + (cu+1) + "].");
}
+
//allocate output frame (incl deep copy schema)
if( ret == null )
ret = new FrameBlock();
- ret._numRows = _numRows;
+ ret._numRows = _numRows;
ret._schema = new ArrayList<ValueType>(_schema);
ret._colnames = new ArrayList<String>(_colnames);
+ ret._colmeta = new ArrayList<ColumnMetadata>(_colmeta);
//copy data to output and partial overwrite w/ rhs
for( int j=0; j<getNumColumns(); j++ ) {
@@ -618,9 +634,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @param ret
* @return
*/
- public FrameBlock sliceOperations(int rl, int ru, int cl, int cu, FrameBlock ret)
+ public FrameBlock sliceOperations(int rl, int ru, int cl, int cu, CacheBlock retCache)
throws DMLRuntimeException
{
+ FrameBlock ret = (FrameBlock)retCache;
// check the validity of bounds
if ( rl < 0 || rl >= getNumRows() || ru < rl || ru >= getNumRows()
|| cl < 0 || cu >= getNumColumns() || cu < cl || cu >= getNumColumns() ) {
@@ -638,6 +655,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
for( int j=cl; j<=cu; j++ ) {
ret._schema.add(_schema.get(j));
ret._colnames.add(_colnames.get(j));
+ ret._colmeta.add(_colmeta.get(j));
}
ret._numRows = ru-rl+1;
@@ -652,6 +670,41 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
return ret;
}
+
+ public void sliceOperations(ArrayList<Pair<Long,FrameBlock>> outlist, IndexRange range, int rowCut)
+ {
+ FrameBlock top=null, bottom=null;
+ Iterator<Pair<Long,FrameBlock>> p=outlist.iterator();
+
+ if(range.rowStart<rowCut)
+ top=(FrameBlock) p.next().getValue();
+
+ if(range.rowEnd>=rowCut)
+ bottom=(FrameBlock) p.next().getValue();
+
+ if(getNumRows() > 0)
+ {
+ int r=(int) range.rowStart;
+
+ for(; r<Math.min(rowCut, range.rowEnd+1); r++)
+ {
+ Object[] row = new Object[(int) (range.colEnd-range.colStart+1)];
+ for(int c=(int) range.colStart; c<range.colEnd+1; c++)
+ row[(int) (c-range.colStart)] = get(r,c);
+ top.appendRow(row);
+ }
+
+ for(; r<=range.rowEnd; r++)
+ {
+ Object[] row = new Object[(int) (range.colEnd-range.colStart+1)];
+ for(int c=(int) range.colStart; c<range.colEnd+1; c++)
+ row[(int) (c-range.colStart)] = get(r,c);
+ bottom.appendRow(row);
+ }
+ }
+
+ }
+
/**
* Appends the given argument frameblock 'that' to this frameblock by
* creating a deep copy to prevent side effects. For cbind, the frames
@@ -789,12 +842,18 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
return map;
}
+ public void merge(CacheBlock that, boolean bDummy)
+ throws DMLRuntimeException
+ {
+ merge((FrameBlock)that);
+ }
+
/**
*
* @param that
* @throws DMLRuntimeException
*/
- public void merge(FrameBlock that )
+ public void merge(FrameBlock that)
throws DMLRuntimeException
{
//check for empty input source (nothing to merge)
@@ -802,35 +861,77 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
return;
//check dimensions (before potentially copy to prevent implicit dimension change)
- if( getNumRows() != that.getNumRows() || getNumColumns() != that.getNumColumns() )
+ if ( getNumRows() != that.getNumRows() || getNumColumns() != that.getNumColumns() )
throw new DMLRuntimeException("Dimension mismatch on merge disjoint (target="+getNumRows()+"x"+getNumColumns()+", source="+that.getNumRows()+"x"+that.getNumColumns()+")");
//core frame block merge through cell copy
- for( int i=0; i<getNumRows(); i++ ) {
+ for( int i=0; i<that.getNumRows(); i++ ) {
for( int j=0; j<getNumColumns(); j++ ) {
- switch( _schema.get(j) ) {
- case STRING:
- if (that.get(i,j) != null)
- set(i,j,that.get(i, j));
- break;
- case BOOLEAN:
- if ((Boolean)that.get(i,j) != Boolean.getBoolean("false"))
- set(i,j,that.get(i, j));
- break;
- case INT:
- if ((Long)that.get(i,j) != 0)
- set(i,j,that.get(i, j));
- break;
- case DOUBLE:
- if ((Double)that.get(i,j) != 0.0)
- set(i,j,that.get(i, j));
- break;
- default: throw new RuntimeException("Unsupported value type: "+_schema.get(j));
- }
+ Object obj = UtilFunctions.objectToObject(getSchema().get(j), that.get(i,j), true);
+ if (obj != null) // Do not update with "null" data
+ set(i, j,obj);
}
}
}
+
+ /**
+ * This function ZERO OUT the data in the slicing window applicable for this block.
+ *
+ *
+ * @param result
+ * @param range
+ * @param complementary
+ * @param iRowStartSrc
+ * @param iRowStartDest
+ * @param brlen
+ * @param iMaxRowsToCopy
+ *
+ */
+ public FrameBlock zeroOutOperations(FrameBlock result, IndexRange range, boolean complementary, int iRowStartSrc, int iRowStartDest, int brlen, int iMaxRowsToCopy)
+ throws DMLRuntimeException
+ {
+ int clen = getNumColumns();
+
+ if(result==null)
+ result=new FrameBlock(getSchema());
+ else
+ {
+ result.reset(0);
+ result.setSchema(getSchema());
+ }
+ result.ensureAllocatedColumns(brlen);
+
+ if(complementary)
+ {
+ for(int r=(int) range.rowStart; r<=range.rowEnd&&r+iRowStartDest<brlen; r++)
+ {
+ for(int c=(int) range.colStart; c<=range.colEnd; c++)
+ result.set(r+iRowStartDest, c, get(r+iRowStartSrc,c));
+ }
+ }else
+ {
+ int r=iRowStartDest;
+ for(; r<(int)range.rowStart && r-iRowStartDest<iMaxRowsToCopy ; r++)
+ for(int c=0; c<clen; c++/*, offset++*/)
+ result.set(r, c, get(r+iRowStartSrc-iRowStartDest,c));
+
+ for(; r<=(int)range.rowEnd && r-iRowStartDest<iMaxRowsToCopy ; r++)
+ {
+ for(int c=0; c<(int)range.colStart; c++)
+ result.set(r, c, get(r+iRowStartSrc-iRowStartDest,c));
+
+ for(int c=(int)range.colEnd+1; c<clen; c++)
+ result.set(r, c, get(r+iRowStartSrc-iRowStartDest,c));
+ }
+
+ for(; r-iRowStartDest<iMaxRowsToCopy ; r++)
+ for(int c=0; c<clen; c++)
+ result.set(r, c, get(r+iRowStartSrc-iRowStartDest,c));
+ }
+
+ return result;
+ }
///////
@@ -887,6 +988,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
}
}
+
/**
*
*/
@@ -949,7 +1051,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
_data[index] = value;
}
public void set(int rl, int ru, Array value) {
- System.arraycopy(((StringArray)value)._data, 0, _data, rl, ru-rl+1);
+ set(rl, ru, value, 0);
}
public void set(int rl, int ru, Array value, int rlSrc) {
System.arraycopy(((StringArray)value)._data, rlSrc, _data, rl, ru-rl+1);
@@ -995,7 +1097,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
_data[index] = (value!=null) ? value : false;
}
public void set(int rl, int ru, Array value) {
- System.arraycopy(((BooleanArray)value)._data, 0, _data, rl, ru-rl+1);
+ set(rl, ru, value, 0);
}
public void set(int rl, int ru, Array value, int rlSrc) {
System.arraycopy(((BooleanArray)value)._data, rlSrc, _data, rl, ru-rl+1);
@@ -1042,7 +1144,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
_data[index] = (value!=null) ? value : 0L;
}
public void set(int rl, int ru, Array value) {
- System.arraycopy(((LongArray)value)._data, 0, _data, rl, ru-rl+1);
+ set(rl, ru, value, 0);
}
public void set(int rl, int ru, Array value, int rlSrc) {
System.arraycopy(((LongArray)value)._data, rlSrc, _data, rl, ru-rl+1);
@@ -1089,7 +1191,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
_data[index] = (value!=null) ? value : 0d;
}
public void set(int rl, int ru, Array value) {
- System.arraycopy(((DoubleArray)value)._data, 0, _data, rl, ru-rl+1);
+ set(rl,ru, value, 0);
}
public void set(int rl, int ru, Array value, int rlSrc) {
System.arraycopy(((DoubleArray)value)._data, rlSrc, _data, rl, ru-rl+1);
@@ -1147,4 +1249,15 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
_mvValue = mvVal;
}
}
+
+ @Override
+ public ArrayList getPairList() {
+ return new ArrayList<Pair<Long, FrameBlock>>();
+ }
+
+ public ArrayList<Pair<?, ?>> performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, CacheBlock in) throws DMLRuntimeException
+ {
+ return OperationsOnMatrixValues.performSlice(ixrange, brlen, bclen, iix, jix, (FrameBlock)in);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 54cba0f..9b40cbd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -86,6 +86,7 @@ import org.apache.sysml.runtime.util.IndexRange;
import org.apache.sysml.runtime.util.UtilFunctions;
+
public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizable
{
private static final long serialVersionUID = 7319972089143154056L;
@@ -1734,6 +1735,12 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
Arrays.fill(denseBlock, ix2, ix2+rowLen, 0);
}
+ public void merge(CacheBlock that, boolean appendOnly)
+ throws DMLRuntimeException
+ {
+ merge((MatrixBlock)that, appendOnly);
+ }
+
/**
* Merge disjoint: merges all non-zero values of the given input into the current
@@ -4034,7 +4041,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
*
* @throws DMLRuntimeException
*/
- public MatrixBlock sliceOperations(int rl, int ru, int cl, int cu, MatrixBlock ret)
+ public MatrixBlock sliceOperations(int rl, int ru, int cl, int cu, CacheBlock ret)
throws DMLRuntimeException
{
// check the validity of bounds
@@ -4046,7 +4053,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
// Output matrix will have the same sparsity as that of the input matrix.
// (assuming a uniform distribution of non-zeros in the input)
- MatrixBlock result=checkType(ret);
+ MatrixBlock result=checkType((MatrixBlock)ret);
long estnnz= (long) ((double)this.nonZeros/rlen/clen*(ru-rl+1)*(cu-cl+1));
boolean result_sparsity = this.sparse && MatrixBlock.evalSparseFormatInMemory(ru-rl+1, cu-cl+1, estnnz);
if(result==null)
@@ -6213,4 +6220,15 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
}
public SparsityEstimate(){}
}
+
+ public ArrayList<Pair<MatrixIndexes, MatrixBlock>> getPairList()
+ {
+ return new ArrayList<Pair<MatrixIndexes,MatrixBlock>>();
+ }
+
+ @SuppressWarnings("unchecked")
+ public ArrayList<Pair<?, ?>> performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, CacheBlock in) throws DMLRuntimeException
+ {
+ return OperationsOnMatrixValues.performSlice(ixrange, brlen, bclen, iix, jix, (MatrixBlock)in);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
index 03a36fe..be12228 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
@@ -21,10 +21,14 @@
package org.apache.sysml.runtime.matrix.data;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
@@ -292,6 +296,17 @@ public class OperationsOnMatrixValues
value1.aggregateBinaryOperations(value1, value2, valueOut, op);
}
+ @SuppressWarnings("rawtypes")
+ public static ArrayList performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, MatrixBlock in)
+ throws DMLRuntimeException
+ {
+ IndexedMatrixValue imv = new IndexedMatrixValue(new MatrixIndexes(iix, jix), (MatrixBlock)in);
+ ArrayList<IndexedMatrixValue> outlist = new ArrayList<IndexedMatrixValue>();
+ performSlice(imv, ixrange, brlen, bclen, outlist);
+
+ return SparkUtils.fromIndexedMatrixBlockToPair(outlist);
+ }
+
/**
*
* @param val
@@ -463,4 +478,144 @@ public class OperationsOnMatrixValues
}
}
}
+
+ @SuppressWarnings("rawtypes")
+ public static ArrayList performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, FrameBlock in)
+ throws DMLRuntimeException
+ {
+ Pair<Long, FrameBlock> lfp = new Pair<Long, FrameBlock>(new Long(((iix-1)*brlen)+1), in);
+ ArrayList<Pair<Long, FrameBlock>> outlist = new ArrayList<Pair<Long, FrameBlock>>();
+ performSlice(lfp, ixrange, brlen, bclen, outlist);
+
+ return outlist;
+ }
+
+
+ /**
+ * This function will get slice of the input frame block overlapping in overall slice(Range), slice has requested for.
+ *
+ * @param val
+ * @param range
+ * @param brlen
+ * @param bclen
+ * @param outlist
+ * @throws DMLRuntimeException
+ */
+ public static void performSlice(Pair<Long,FrameBlock> in, IndexRange ixrange, int brlen, int bclen, ArrayList<Pair<Long,FrameBlock>> outlist)
+ throws DMLRuntimeException
+ {
+ long index = in.getKey();
+ FrameBlock block = in.getValue();
+
+ // Get Block indexes (rows and columns boundaries)
+ long cellIndexTopRow = index;
+ long cellIndexBottomRow = index+block.getNumRows()-1;
+ long cellIndexLeftCol = 1;
+ long cellIndexRightCol = block.getNumColumns();
+
+ // Calculate block boundaries with range of slice to be performed (Global index)
+ long cellIndexOverlapTop = Math.max(cellIndexTopRow, ixrange.rowStart);
+ long cellIndexOverlapBottom = Math.min(cellIndexBottomRow, ixrange.rowEnd);
+ long cellIndexOverlapLeft = Math.max(cellIndexLeftCol, ixrange.colStart);
+ long cellIndexOverlapRight = Math.min(cellIndexRightCol, ixrange.colEnd);
+
+ //check if block is outside the indexing range
+ if(cellIndexOverlapTop>cellIndexOverlapBottom || cellIndexOverlapLeft>cellIndexOverlapRight) {
+ return;
+ }
+
+ // Create IndexRange for the slice to be performed on this block.
+ IndexRange tmpRange = new IndexRange(
+ cellIndexOverlapTop - index,
+ cellIndexOverlapBottom - index,
+ cellIndexOverlapLeft -1,
+ cellIndexOverlapRight - 1);
+
+ // Get Top Row and Left column cutting point.
+ int rowCut=(int)(ixrange.rowStart-index);
+
+ // Get indices for result block
+ long resultBlockIndexTop=UtilFunctions.computeBlockIndex(cellIndexOverlapTop, brlen);
+ long resultBlockIndexBottom=UtilFunctions.computeBlockIndex(cellIndexOverlapBottom, brlen);
+
+ //allocate space for the output value
+ for(long r=resultBlockIndexTop; r<=resultBlockIndexBottom; r++)
+ {
+ List<ValueType> schema = UtilFunctions.getSubSchema(block.getSchema(), tmpRange.colStart, tmpRange.colEnd);
+ long iResultIndex = (r-1)*brlen+tmpRange.rowStart;
+ Pair<Long,FrameBlock> out=new Pair<Long,FrameBlock>(new Long(iResultIndex+1), new FrameBlock(schema));
+ outlist.add(out);
+ }
+
+ //execute actual slice operation
+ block.sliceOperations(outlist, tmpRange, rowCut);
+ }
+
+ /**
+ *
+ * @param in
+ * @param ixrange
+ * @param brlen
+ * @param bclen
+ * @param rlen
+ * @param clen
+ * @param outlist
+ * @throws DMLRuntimeException
+ */
+ public static void performShift(Pair<Long,FrameBlock> in, IndexRange ixrange, int brlenLeft, int clenLeft/*, int bclen*/, long rlen, long clen, ArrayList<Pair<Long,FrameBlock>> outlist)
+ throws DMLRuntimeException
+ {
+ Long ix = in.getKey();
+ FrameBlock fb = in.getValue();
+ long start_lhs_globalRowIndex = ixrange.rowStart + (ix-1);
+ long start_lhs_globalColIndex = ixrange.colStart;
+ long end_lhs_globalRowIndex = start_lhs_globalRowIndex + fb.getNumRows() - 1;
+ long end_lhs_globalColIndex = ixrange.colEnd;
+
+ long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlenLeft);
+ long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlenLeft);
+
+ for(long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
+
+ // Calculate global index of right hand side block
+ long lhs_rl = Math.max((leftRowIndex-1)*brlenLeft+1, start_lhs_globalRowIndex);
+ long lhs_ru = Math.min(leftRowIndex*brlenLeft, end_lhs_globalRowIndex);
+ long lhs_cl = start_lhs_globalColIndex;
+ long lhs_cu = end_lhs_globalColIndex;
+
+ int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlenLeft);
+ int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlenLeft);
+ int lhs_lcl = (int)lhs_cl-1;
+ int lhs_lcu = (int)lhs_cu-1;
+
+ long rhs_rl = lhs_rl - (ixrange.rowStart-1) - (ix-1);
+ long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
+ long rhs_cl = lhs_cl - ixrange.colStart + 1;
+ long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);
+
+ // local indices are 0 (zero) based.
+ int rhs_lrl = (int) (UtilFunctions.computeCellInBlock(rhs_rl, fb.getNumRows()));
+ int rhs_lru = (int) (UtilFunctions.computeCellInBlock(rhs_ru, fb.getNumRows()));
+ int rhs_lcl = (int)rhs_cl-1;
+ int rhs_lcu = (int)rhs_cu-1;
+
+ FrameBlock slicedRHSBlk = fb.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new FrameBlock());
+
+ int lbclen = clenLeft;
+
+ List<ValueType> schemaPartialLeft = Collections.nCopies(lhs_lcl, ValueType.STRING);
+ List<ValueType> schemaRHS = UtilFunctions.getSubSchema(fb.getSchema(), rhs_lcl, rhs_lcl-lhs_lcl+lhs_lcu);
+ List<ValueType> schema = new ArrayList<ValueType>(schemaPartialLeft);
+ schema.addAll(schemaRHS);
+ List<ValueType> schemaPartialRight = Collections.nCopies(lbclen-schema.size(), ValueType.STRING);
+ schema.addAll(schemaPartialRight);
+ FrameBlock resultBlock = new FrameBlock(schema);
+ int iRHSRows = (int)(leftRowIndex<=rlen/brlenLeft?brlenLeft:rlen-(rlen/brlenLeft)*brlenLeft);
+ resultBlock.ensureAllocatedColumns(iRHSRows);
+
+ resultBlock = resultBlock.leftIndexingOperations(slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, new FrameBlock());
+ outlist.add(new Pair<Long, FrameBlock>((leftRowIndex-1)*brlenLeft+1, resultBlock));
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java
index 423607c..4d4a975 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OutputInfo.java
@@ -92,7 +92,7 @@ public class OutputInfo implements Serializable
else
throw new DMLRuntimeException("Unrecognized output info: " + oi);
}
-
+
public static OutputInfo stringToOutputInfo (String str) {
if ( str.equalsIgnoreCase("textcell")) {
return TextCellOutputInfo;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/266d4c8c/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
index e7aebb4..b059a06 100644
--- a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
+++ b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
@@ -201,7 +201,8 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements
@Override
public void writeUTF(String s) throws IOException {
- throw new IOException("Not supported.");
+ byte[] strBytes = s.getBytes("UTF-8");
+ write(strBytes, 0, strBytes.length);
}