You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ni...@apache.org on 2017/02/17 05:43:26 UTC
incubator-systemml git commit: [MINOR] Addressed corner cases (i.e.
empty blocks and extremely large blocks) as well added recomputing nnz in the
converters
Repository: incubator-systemml
Updated Branches:
refs/heads/master 0c85c1e52 -> 7e2383bfa
[MINOR] Addressed corner cases (i.e. empty blocks and extremely large
blocks) as well added recomputing nnz in the converters
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7e2383bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7e2383bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7e2383bf
Branch: refs/heads/master
Commit: 7e2383bfa51fc51cfe637680f94758eb1ae193cf
Parents: 0c85c1e
Author: Niketan Pansare <np...@us.ibm.com>
Authored: Thu Feb 16 21:41:07 2017 -0800
Committer: Niketan Pansare <np...@us.ibm.com>
Committed: Thu Feb 16 21:42:05 2017 -0800
----------------------------------------------------------------------
.../spark/utils/RDDConverterUtilsExt.java | 34 ++++++++++++++------
1 file changed, 25 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7e2383bf/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
index 973db64..dea5601 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
@@ -154,8 +154,10 @@ public class RDDConverterUtilsExt
double val = buf1.getDouble();
int rowIndex = buf2.getInt();
int colIndex = buf3.getInt();
- mb.setValue(rowIndex, colIndex, val); // TODO: Improve the performance
+ mb.setValue(rowIndex, colIndex, val);
}
+ mb.recomputeNonZeros();
+ mb.examSparsity();
return mb;
}
@@ -169,7 +171,10 @@ public class RDDConverterUtilsExt
throw new DMLRuntimeException("Convertion to sparse format not supported");
}
else {
- double [] denseBlock = new double[rlen*clen];
+ long limit = rlen*clen;
+ if( limit > Integer.MAX_VALUE )
+ throw new DMLRuntimeException("Dense NumPy array of size " + limit + " cannot be converted to MatrixBlock");
+ double [] denseBlock = new double[(int) limit];
ByteBuffer buf = ByteBuffer.wrap(data);
buf.order(ByteOrder.nativeOrder());
for(int i = 0; i < rlen*clen; i++) {
@@ -177,6 +182,7 @@ public class RDDConverterUtilsExt
}
mb.init( denseBlock, rlen, clen );
}
+ mb.recomputeNonZeros();
mb.examSparsity();
return mb;
}
@@ -185,17 +191,27 @@ public class RDDConverterUtilsExt
byte [] ret = null;
if(mb.isInSparseFormat()) {
mb.sparseToDense();
-// throw new DMLRuntimeException("Sparse to dense conversion is not yet implemented");
}
+ long limit = mb.getNumRows()*mb.getNumColumns();
+ int times = Double.SIZE / Byte.SIZE;
+ if( limit * times > Integer.MAX_VALUE )
+ throw new DMLRuntimeException("MatrixBlock of size " + limit + " cannot be converted to dense numpy array");
+ ret = new byte[(int) (limit * times)];
+
double [] denseBlock = mb.getDenseBlock();
- if(denseBlock == null) {
- throw new DMLRuntimeException("Sparse to dense conversion is not yet implemented");
+ if(mb.isEmptyBlock()) {
+ for(int i=0;i < limit;i++){
+ ByteBuffer.wrap(ret, i*times, times).order(ByteOrder.nativeOrder()).putDouble(0);
+ }
}
- int times = Double.SIZE / Byte.SIZE;
- ret = new byte[denseBlock.length * times];
- for(int i=0;i < denseBlock.length;i++){
- ByteBuffer.wrap(ret, i*times, times).order(ByteOrder.nativeOrder()).putDouble(denseBlock[i]);
+ else if(denseBlock == null) {
+ throw new DMLRuntimeException("Error while dealing with empty blocks.");
+ }
+ else {
+ for(int i=0;i < denseBlock.length;i++){
+ ByteBuffer.wrap(ret, i*times, times).order(ByteOrder.nativeOrder()).putDouble(denseBlock[i]);
+ }
}
return ret;