You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/04/06 22:04:56 UTC
systemml git commit: [HOTFIX][SYSTEMML-2235] Fix bufferpool
serialization compressed blocks
Repository: systemml
Updated Branches:
refs/heads/master f700df2c8 -> 68a7b44b5
[HOTFIX][SYSTEMML-2235] Fix bufferpool serialization compressed blocks
The recent fix of a bufferpool leak in SYSTEMML-2234 also made a small
improvement to the buffer pool threshold handling, which revealed a
hidden bug of missing bufferpool serialization of compressed matrix
blocks. This patch fixes this issue by correctly handling the shallow
serialize meta data, implementing missing char serialization
functionality, and fixing the reported exact size of compressed blocks
in serialized form (wrong header size).
Furthermore, this patch also makes some refactoring to eliminate
redundancy in the cache data and buffered data streams.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/68a7b44b
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/68a7b44b
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/68a7b44b
Branch: refs/heads/master
Commit: 68a7b44b5c6e783dd9d5113d3ee03b1b1a019afa
Parents: f700df2
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Apr 6 14:53:13 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Apr 6 15:06:16 2018 -0700
----------------------------------------------------------------------
.../runtime/compress/CompressedMatrixBlock.java | 28 ++-
.../controlprogram/caching/CacheDataInput.java | 53 ++----
.../controlprogram/caching/CacheDataOutput.java | 65 +++----
.../sysml/runtime/io/IOUtilFunctions.java | 52 ++++++
.../util/FastBufferedDataInputStream.java | 82 +++------
.../util/FastBufferedDataOutputStream.java | 171 +++++++------------
.../codegen/CompressedOuterProductTest.java | 8 +-
7 files changed, 204 insertions(+), 255 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
index 569ef14..2051f34 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
@@ -751,7 +751,7 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
timePhase4 = t4;
timePhase5 = t5;
}
- }
+ }
@Override
public double quickGetValue(int r, int c) {
@@ -768,26 +768,38 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
//find row value
return grp.get(r, c);
- }
+ }
//////////////////////////////////////////
// Serialization / Deserialization
@Override
- public long getExactSizeOnDisk()
- {
+ public long getExactSizeOnDisk() {
//header information
- long ret = 12;
-
+ long ret = 22;
for( ColGroup grp : _colGroups ) {
ret += 1; //type info
ret += grp.getExactSizeOnDisk();
}
-
return ret;
}
@Override
+ public boolean isShallowSerialize() {
+ return false;
+ }
+
+ @Override
+ public boolean isShallowSerialize(boolean inclConvert) {
+ return false;
+ }
+
+ @Override
+ public void toShallowSerializeBlock() {
+ //do nothing
+ }
+
+ @Override
public void readFields(DataInput in)
throws IOException
{
@@ -980,7 +992,7 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
ret2 = (CompressedMatrixBlock) ret;
ret2.reset(m, n);
}
-
+
//shallow copy of lhs column groups
ret2.allocateColGroupList();
ret2._colGroups.addAll(_colGroups);
http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java
index 8f16ae1..dde2211 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java
@@ -22,18 +22,17 @@ package org.apache.sysml.runtime.controlprogram.caching;
import java.io.DataInput;
import java.io.IOException;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.data.MatrixBlockDataInput;
import org.apache.sysml.runtime.matrix.data.SparseBlock;
public class CacheDataInput implements DataInput, MatrixBlockDataInput
{
-
protected byte[] _buff;
protected int _bufflen;
protected int _count;
- public CacheDataInput( byte[] mem )
- {
+ public CacheDataInput( byte[] mem ) {
_buff = mem;
_bufflen = _buff.length;
_count = 0;
@@ -55,17 +54,13 @@ public class CacheDataInput implements DataInput, MatrixBlockDataInput
}
@Override
- public boolean readBoolean()
- throws IOException
- {
+ public boolean readBoolean() throws IOException {
//mask to adhere to the input stream semantic
return ( (_buff[_count++] & 0xFF) != 0 );
}
@Override
- public byte readByte()
- throws IOException
- {
+ public byte readByte() throws IOException {
//mask to adhere to the input stream semantic
return (byte) (_buff[_count++] & 0xFF);
}
@@ -87,26 +82,22 @@ public class CacheDataInput implements DataInput, MatrixBlockDataInput
@Override
public char readChar() throws IOException {
- throw new IOException("Not supported.");
+ int ret = baToShort(_buff, _count);
+ _count += 2;
+ return (char) ret;
}
@Override
- public int readInt()
- throws IOException
- {
+ public int readInt() throws IOException {
int ret = baToInt(_buff, _count);
_count += 4;
-
return ret;
}
@Override
- public long readLong()
- throws IOException
- {
+ public long readLong() throws IOException {
long ret = baToLong(_buff, _count);
_count += 8;
-
return ret;
}
@@ -200,25 +191,15 @@ public class CacheDataInput implements DataInput, MatrixBlockDataInput
return nnz;
}
- private static int baToInt( byte[] ba, final int off )
- {
- //shift and add 4 bytes into single int
- return ((ba[off+0] & 0xFF) << 24) +
- ((ba[off+1] & 0xFF) << 16) +
- ((ba[off+2] & 0xFF) << 8) +
- ((ba[off+3] & 0xFF) << 0);
+ private static int baToShort( byte[] ba, final int off ) {
+ return IOUtilFunctions.baToShort(ba, off);
}
- private static long baToLong( byte[] ba, final int off )
- {
- //shift and add 8 bytes into single long
- return ((long)(ba[off+0] & 0xFF) << 56) +
- ((long)(ba[off+1] & 0xFF) << 48) +
- ((long)(ba[off+2] & 0xFF) << 40) +
- ((long)(ba[off+3] & 0xFF) << 32) +
- ((long)(ba[off+4] & 0xFF) << 24) +
- ((long)(ba[off+5] & 0xFF) << 16) +
- ((long)(ba[off+6] & 0xFF) << 8) +
- ((long)(ba[off+7] & 0xFF) << 0);
+ private static int baToInt( byte[] ba, final int off ) {
+ return IOUtilFunctions.baToInt(ba, off);
+ }
+
+ private static long baToLong( byte[] ba, final int off ) {
+ return IOUtilFunctions.baToLong(ba, off);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java
index b9ed0a8..1b740d2 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java
@@ -37,7 +37,7 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput
protected int _bufflen;
protected int _count;
- public CacheDataOutput( byte[] mem ) {
+ public CacheDataOutput( byte[] mem ) {
_buff = mem;
_bufflen = _buff.length;
_count = 0;
@@ -46,26 +46,25 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput
@Override
public void write(int b) throws IOException {
_buff[_count++] = (byte)b;
- }
+ }
- @Override
+ @Override
public void write(byte[] b) throws IOException {
System.arraycopy(b, 0, _buff, _count, b.length);
_count += b.length;
}
-
- @Override
+
+ @Override
public void write(byte[] b, int off, int len) throws IOException {
System.arraycopy(b, off, _buff, _count, len);
_count += len;
- }
+ }
@Override
public void writeBoolean(boolean v) throws IOException {
_buff[_count++] = (byte)( v ? 1 : 0 );
}
-
@Override
public void writeInt(int v) throws IOException {
intToBa(v, _buff, _count);
@@ -74,14 +73,14 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput
@Override
public void writeDouble(double v) throws IOException {
- long tmp = Double.doubleToRawLongBits(v);
+ long tmp = Double.doubleToRawLongBits(v);
longToBa(tmp, _buff, _count);
_count += 8;
}
@Override
public void writeByte(int v) throws IOException {
- _buff[_count++] = (byte) v;
+ _buff[_count++] = (byte) v;
}
@Override
@@ -91,7 +90,7 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput
@Override
public void writeChar(int v) throws IOException {
- throw new IOException("Not supported.");
+ writeShort(v);
}
@Override
@@ -106,7 +105,7 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput
@Override
public void writeLong(long v) throws IOException {
- longToBa(v, _buff, _count);
+ longToBa(v, _buff, _count);
_count += 8;
}
@@ -143,10 +142,10 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput
}
}
- ///////////////////////////////////////////////
- // Implementation of MatrixBlockDSMDataOutput
- ///////////////////////////////////////////////
-
+ ///////////////////////////////////////////////
+ // Implementation of MatrixBlockDSMDataOutput
+ ///////////////////////////////////////////////
+
@Override
public void writeDoubleArray(int len, double[] varr)
throws IOException
@@ -155,10 +154,9 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput
int off = _count;
//serialize entire array into buffer
- for( int i=0; i<len; i++ )
- {
- long tmp = Double.doubleToRawLongBits(varr[i]);
- longToBa(tmp, _buff, off+i*8);
+ for( int i=0; i<len; i++ ) {
+ long tmp = Double.doubleToRawLongBits(varr[i]);
+ longToBa(tmp, _buff, off+i*8);
}
//update buffer offset
@@ -200,32 +198,15 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput
writeInt( 0 );
}
- private static void shortToBa( final int val, byte[] ba, final int off )
- {
- //shift and mask out 2 bytes
- ba[ off+0 ] = (byte)((val >>> 8) & 0xFF);
- ba[ off+1 ] = (byte)((val >>> 0) & 0xFF);
+ private static void shortToBa( final int val, byte[] ba, final int off ) {
+ IOUtilFunctions.shortToBa(val, ba, off);
}
- private static void intToBa( final int val, byte[] ba, final int off )
- {
- //shift and mask out 4 bytes
- ba[ off+0 ] = (byte)((val >>> 24) & 0xFF);
- ba[ off+1 ] = (byte)((val >>> 16) & 0xFF);
- ba[ off+2 ] = (byte)((val >>> 8) & 0xFF);
- ba[ off+3 ] = (byte)((val >>> 0) & 0xFF);
+ private static void intToBa( final int val, byte[] ba, final int off ) {
+ IOUtilFunctions.intToBa(val, ba, off);
}
- private static void longToBa( final long val, byte[] ba, final int off )
- {
- //shift and mask out 8 bytes
- ba[ off+0 ] = (byte)((val >>> 56) & 0xFF);
- ba[ off+1 ] = (byte)((val >>> 48) & 0xFF);
- ba[ off+2 ] = (byte)((val >>> 40) & 0xFF);
- ba[ off+3 ] = (byte)((val >>> 32) & 0xFF);
- ba[ off+4 ] = (byte)((val >>> 24) & 0xFF);
- ba[ off+5 ] = (byte)((val >>> 16) & 0xFF);
- ba[ off+6 ] = (byte)((val >>> 8) & 0xFF);
- ba[ off+7 ] = (byte)((val >>> 0) & 0xFF);
+ private static void longToBa( final long val, byte[] ba, final int off ) {
+ IOUtilFunctions.longToBa(val, ba, off);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
index 4d388c9..b484dfc 100644
--- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
@@ -513,4 +513,56 @@ public class IOUtilFunctions
fs.delete(fnameMtdCrc, false);
}
}
+
+ public static int baToShort( byte[] ba, final int off ) {
+ //shift and add 2 bytes into single int
+ return ((ba[off+0] & 0xFF) << 8)
+ + ((ba[off+1] & 0xFF) << 0);
+ }
+
+ public static int baToInt( byte[] ba, final int off ) {
+ //shift and add 4 bytes into single int
+ return ((ba[off+0] & 0xFF) << 24)
+ + ((ba[off+1] & 0xFF) << 16)
+ + ((ba[off+2] & 0xFF) << 8)
+ + ((ba[off+3] & 0xFF) << 0);
+ }
+
+ public static long baToLong( byte[] ba, final int off ) {
+ //shift and add 8 bytes into single long
+ return ((long)(ba[off+0] & 0xFF) << 56)
+ + ((long)(ba[off+1] & 0xFF) << 48)
+ + ((long)(ba[off+2] & 0xFF) << 40)
+ + ((long)(ba[off+3] & 0xFF) << 32)
+ + ((long)(ba[off+4] & 0xFF) << 24)
+ + ((long)(ba[off+5] & 0xFF) << 16)
+ + ((long)(ba[off+6] & 0xFF) << 8)
+ + ((long)(ba[off+7] & 0xFF) << 0);
+ }
+
+ public static void shortToBa( final int val, byte[] ba, final int off ) {
+ //shift and mask out 2 bytes
+ ba[ off+0 ] = (byte)((val >>> 8) & 0xFF);
+ ba[ off+1 ] = (byte)((val >>> 0) & 0xFF);
+ }
+
+ public static void intToBa( final int val, byte[] ba, final int off ) {
+ //shift and mask out 4 bytes
+ ba[ off+0 ] = (byte)((val >>> 24) & 0xFF);
+ ba[ off+1 ] = (byte)((val >>> 16) & 0xFF);
+ ba[ off+2 ] = (byte)((val >>> 8) & 0xFF);
+ ba[ off+3 ] = (byte)((val >>> 0) & 0xFF);
+ }
+
+ public static void longToBa( final long val, byte[] ba, final int off ) {
+ //shift and mask out 8 bytes
+ ba[ off+0 ] = (byte)((val >>> 56) & 0xFF);
+ ba[ off+1 ] = (byte)((val >>> 48) & 0xFF);
+ ba[ off+2 ] = (byte)((val >>> 40) & 0xFF);
+ ba[ off+3 ] = (byte)((val >>> 32) & 0xFF);
+ ba[ off+4 ] = (byte)((val >>> 24) & 0xFF);
+ ba[ off+5 ] = (byte)((val >>> 16) & 0xFF);
+ ba[ off+6 ] = (byte)((val >>> 8) & 0xFF);
+ ba[ off+7 ] = (byte)((val >>> 0) & 0xFF);
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java
index 197467d..9069859 100644
--- a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java
+++ b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java
@@ -25,13 +25,13 @@ import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.data.MatrixBlockDataInput;
import org.apache.sysml.runtime.matrix.data.SparseBlock;
import org.apache.sysml.runtime.matrix.data.SparseBlockCSR;
public class FastBufferedDataInputStream extends FilterInputStream implements DataInput, MatrixBlockDataInput
{
-
protected byte[] _buff;
protected int _bufflen;
@@ -41,9 +41,8 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da
public FastBufferedDataInputStream( InputStream in, int size ) {
super(in);
-
if (size <= 0)
- throw new IllegalArgumentException("Buffer size <= 0");
+ throw new IllegalArgumentException("Buffer size <= 0");
_buff = new byte[ size ];
_bufflen = size;
}
@@ -52,26 +51,21 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da
// DataInput Implementation
/////////////////////////////
-
@Override
- public void readFully(byte[] b)
- throws IOException
- {
+ public void readFully(byte[] b) throws IOException {
readFully(b, 0, b.length);
}
@Override
- public void readFully(byte[] b, int off, int len)
- throws IOException
- {
+ public void readFully(byte[] b, int off, int len) throws IOException {
if (len < 0)
- throw new IndexOutOfBoundsException();
+ throw new IndexOutOfBoundsException();
int n = 0;
while (n < len) {
- int count = in.read(b, off + n, len - n);
- if (count < 0)
- throw new EOFException();
- n += count;
+ int count = in.read(b, off + n, len - n);
+ if (count < 0)
+ throw new EOFException();
+ n += count;
}
}
@@ -81,16 +75,12 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da
}
@Override
- public boolean readBoolean()
- throws IOException
- {
- return ( in.read() != 0 );
+ public boolean readBoolean() throws IOException {
+ return in.read() != 0;
}
@Override
- public byte readByte()
- throws IOException
- {
+ public byte readByte() throws IOException {
return (byte)in.read();
}
@@ -111,24 +101,19 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da
@Override
public char readChar() throws IOException {
- throw new IOException("Not supported.");
+ readFully(_buff, 0, 2);
+ return (char)baToShort(_buff, 0);
}
@Override
- public int readInt()
- throws IOException
- {
+ public int readInt() throws IOException {
readFully(_buff, 0, 4);
-
return baToInt(_buff, 0);
}
@Override
- public long readLong()
- throws IOException
- {
+ public long readLong() throws IOException {
readFully(_buff, 0, 8);
-
return baToLong(_buff, 0);
}
@@ -138,11 +123,8 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da
}
@Override
- public double readDouble()
- throws IOException
- {
+ public double readDouble() throws IOException {
readFully(_buff, 0, 8);
-
long tmp = baToLong(_buff, 0);
return Double.longBitsToDouble( tmp );
}
@@ -160,7 +142,7 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da
///////////////////////////////////////////////
// Implementation of MatrixBlockDataInput
- ///////////////////////////////////////////////
+ ///////////////////////////////////////////////
@Override
public long readDoubleArray(int len, double[] varr)
@@ -244,7 +226,7 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da
}
}
- gnnz += lnnz;
+ gnnz += lnnz;
}
}
@@ -255,25 +237,15 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da
return nnz;
}
- private static int baToInt( byte[] ba, final int off )
- {
- //shift and add 4 bytes into single int
- return ((ba[off+0] & 0xFF) << 24) +
- ((ba[off+1] & 0xFF) << 16) +
- ((ba[off+2] & 0xFF) << 8) +
- ((ba[off+3] & 0xFF) << 0);
+ private static int baToShort( byte[] ba, final int off ) {
+ return IOUtilFunctions.baToShort(ba, off);
}
- private static long baToLong( byte[] ba, final int off )
- {
- //shift and add 8 bytes into single long
- return ((long)(ba[off+0] & 0xFF) << 56) +
- ((long)(ba[off+1] & 0xFF) << 48) +
- ((long)(ba[off+2] & 0xFF) << 40) +
- ((long)(ba[off+3] & 0xFF) << 32) +
- ((long)(ba[off+4] & 0xFF) << 24) +
- ((long)(ba[off+5] & 0xFF) << 16) +
- ((long)(ba[off+6] & 0xFF) << 8) +
- ((long)(ba[off+7] & 0xFF) << 0);
+ private static int baToInt( byte[] ba, final int off ) {
+ return IOUtilFunctions.baToInt(ba, off);
+ }
+
+ private static long baToLong( byte[] ba, final int off ) {
+ return IOUtilFunctions.baToLong(ba, off);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
index 6173587..dd70d8f 100644
--- a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
+++ b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java
@@ -45,34 +45,27 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements
protected byte[] _buff;
protected int _bufflen;
protected int _count;
-
- public FastBufferedDataOutputStream(OutputStream out)
- {
+
+ public FastBufferedDataOutputStream(OutputStream out) {
this(out, 8192);
}
- public FastBufferedDataOutputStream(OutputStream out, int size)
- {
+ public FastBufferedDataOutputStream(OutputStream out, int size) {
super(out);
-
- if(size <= 0)
- throw new IllegalArgumentException("Buffer size <= 0.");
- if( size%8 != 0 )
- throw new IllegalArgumentException("Buffer size not a multiple of 8.");
-
+ if(size <= 0)
+ throw new IllegalArgumentException("Buffer size <= 0.");
+ if( size%8 != 0 )
+ throw new IllegalArgumentException("Buffer size not a multiple of 8.");
_buff = new byte[size];
_bufflen = size;
}
@Override
- public void write(int b)
- throws IOException
- {
- if (_count >= _bufflen) {
- flushBuffer();
- }
+ public void write(int b) throws IOException {
+ if (_count >= _bufflen)
+ flushBuffer();
_buff[_count++] = (byte)b;
- }
+ }
@Override
public void write(byte[] b, int off, int len)
@@ -90,99 +83,74 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements
_count += len;
}
- @Override
- public void flush()
- throws IOException
- {
- flushBuffer();
- out.flush();
- }
-
- private void flushBuffer()
- throws IOException
- {
- if(_count > 0)
- {
- out.write(_buff, 0, _count);
- _count = 0;
- }
- }
-
- @Override
- public void close()
- throws IOException
- {
- super.close();
- }
-
- /////////////////////////////
- // DataOutput Implementation
- /////////////////////////////
-
@Override
- public void writeBoolean(boolean v)
- throws IOException
- {
- if (_count >= _bufflen) {
- flushBuffer();
+ public void flush() throws IOException {
+ flushBuffer();
+ out.flush();
+ }
+
+ private void flushBuffer() throws IOException {
+ if(_count > 0) {
+ out.write(_buff, 0, _count);
+ _count = 0;
}
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ }
+
+ /////////////////////////////
+ // DataOutput Implementation
+ /////////////////////////////
+
+ @Override
+ public void writeBoolean(boolean v) throws IOException {
+ if (_count >= _bufflen)
+ flushBuffer();
_buff[_count++] = (byte)(v ? 1 : 0);
}
@Override
- public void writeInt(int v)
- throws IOException
- {
- if (_count+4 > _bufflen) {
- flushBuffer();
- }
-
+ public void writeInt(int v) throws IOException {
+ if (_count+4 > _bufflen)
+ flushBuffer();
intToBa(v, _buff, _count);
_count += 4;
}
-
@Override
- public void writeLong(long v)
- throws IOException
- {
- if (_count+8 > _bufflen) {
- flushBuffer();
- }
-
+ public void writeLong(long v) throws IOException {
+ if (_count+8 > _bufflen)
+ flushBuffer();
longToBa(v, _buff, _count);
_count += 8;
}
@Override
- public void writeDouble(double v)
- throws IOException
- {
- if (_count+8 > _bufflen) {
- flushBuffer();
- }
-
- long tmp = Double.doubleToRawLongBits(v);
+ public void writeDouble(double v) throws IOException {
+ if (_count+8 > _bufflen)
+ flushBuffer();
+ long tmp = Double.doubleToRawLongBits(v);
longToBa(tmp, _buff, _count);
_count += 8;
}
@Override
public void writeByte(int v) throws IOException {
- if (_count+1 > _bufflen) {
- flushBuffer();
- }
- _buff[_count++] = (byte) v;
+ if (_count+1 > _bufflen)
+ flushBuffer();
+ _buff[_count++] = (byte) v;
}
@Override
public void writeShort(int v) throws IOException {
- if (_count+2 > _bufflen) {
- flushBuffer();
- }
+ if (_count+2 > _bufflen)
+ flushBuffer();
shortToBa(v, _buff, _count);
- _count += 2;
+ _count += 2;
}
@Override
@@ -192,7 +160,7 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements
@Override
public void writeChar(int v) throws IOException {
- throw new IOException("Not supported.");
+ writeShort(v);
}
@Override
@@ -234,9 +202,9 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements
}
}
- ///////////////////////////////////////////////
- // Implementation of MatrixBlockDSMDataOutput
- ///////////////////////////////////////////////
+ ///////////////////////////////////////////////
+ // Implementation of MatrixBlockDSMDataOutput
+ ///////////////////////////////////////////////
@Override
public void writeDoubleArray(int len, double[] varr)
@@ -320,32 +288,15 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements
writeInt( 0 );
}
- private static void shortToBa( final int val, byte[] ba, final int off )
- {
- //shift and mask out 2 bytes
- ba[ off+0 ] = (byte)((val >>> 8) & 0xFF);
- ba[ off+1 ] = (byte)((val >>> 0) & 0xFF);
+ private static void shortToBa( final int val, byte[] ba, final int off ) {
+ IOUtilFunctions.shortToBa(val, ba, off);
}
- private static void intToBa( final int val, byte[] ba, final int off )
- {
- //shift and mask out 4 bytes
- ba[ off+0 ] = (byte)((val >>> 24) & 0xFF);
- ba[ off+1 ] = (byte)((val >>> 16) & 0xFF);
- ba[ off+2 ] = (byte)((val >>> 8) & 0xFF);
- ba[ off+3 ] = (byte)((val >>> 0) & 0xFF);
+ private static void intToBa( final int val, byte[] ba, final int off ) {
+ IOUtilFunctions.intToBa(val, ba, off);
}
- private static void longToBa( final long val, byte[] ba, final int off )
- {
- //shift and mask out 8 bytes
- ba[ off+0 ] = (byte)((val >>> 56) & 0xFF);
- ba[ off+1 ] = (byte)((val >>> 48) & 0xFF);
- ba[ off+2 ] = (byte)((val >>> 40) & 0xFF);
- ba[ off+3 ] = (byte)((val >>> 32) & 0xFF);
- ba[ off+4 ] = (byte)((val >>> 24) & 0xFF);
- ba[ off+5 ] = (byte)((val >>> 16) & 0xFF);
- ba[ off+6 ] = (byte)((val >>> 8) & 0xFF);
- ba[ off+7 ] = (byte)((val >>> 0) & 0xFF);
+ private static void longToBa( final long val, byte[] ba, final int off ) {
+ IOUtilFunctions.longToBa(val, ba, off);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java
index a71294a..a40b0b7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java
@@ -35,7 +35,7 @@ import org.apache.sysml.test.integration.TestConfiguration;
import org.apache.sysml.test.utils.TestUtils;
public class CompressedOuterProductTest extends AutomatedTestBase
-{
+{
private static final String TEST_NAME1 = "CompressedOuterProductMain";
private static final String TEST_DIR = "functions/codegen/";
private static final String TEST_CLASS_DIR = TEST_DIR + CompressedOuterProductTest.class.getSimpleName() + "/";
@@ -210,11 +210,11 @@ public class CompressedOuterProductTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats",
+ programArgs = new String[]{"-explain", "-stats",
"-args", input("X"), output("R") };
fullRScriptName = HOME + testname + ".R";
- rCmd = getRCmd(inputDir(), expectedDir());
+ rCmd = getRCmd(inputDir(), expectedDir());
//generate input data
double sparsity = -1;
@@ -254,7 +254,7 @@ public class CompressedOuterProductTest extends AutomatedTestBase
OptimizerUtils.ALLOW_OPERATOR_FUSION = true;
CompressedMatrixBlock.ALLOW_DDC_ENCODING = true;
}
- }
+ }
/**
* Override default configuration with custom test configuration to ensure