[systemds] branch master updated: [MINOR] Check if NNZ is in mdt before parsing

[ba...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new 9c0d0b7  [MINOR] Check if NNZ is in mdt before parsing
9c0d0b7 is described below

commit 9c0d0b731be06f3112661a9c04740b91a3870e7e
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Wed Aug 26 16:43:02 2020 +0200

    [MINOR] Check if NNZ is in mdt before parsing
    
    This commit change the behavior of a federated worker if the
    metadata file does not contain the number of non zeros.
    It makes it default to parse into a dense representation when the input
    is small, and sparse representation if large.
---
 .../controlprogram/federated/FederatedWorkerHandler.java    | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
index fb8c494..5dbccb4 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
@@ -195,7 +195,18 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
 							new FederatedWorkerHandlerException("Could not parse metadata file"));
 					mc.setRows(mtd.getLong(DataExpression.READROWPARAM));
 					mc.setCols(mtd.getLong(DataExpression.READCOLPARAM));
-					mc.setNonZeros(mtd.getLong(DataExpression.READNNZPARAM));
+					if(mtd.containsKey(DataExpression.READNNZPARAM)){
+						mc.setNonZeros(mtd.getLong(DataExpression.READNNZPARAM));
+					}
+					else if (mc.getCols() * mc.getRows() < 8000000){
+						// force dense allocation.
+						mc.setNonZeros(mc.getCols() *mc.getRows());
+					}
+					else{
+						// force sparse allocation 
+						mc.setNonZeros((long)(mc.getCols() * mc.getRows() * 0.35));
+					}
+					
 					cd = (CacheableData<?>) PrivacyPropagator.parseAndSetPrivacyConstraint(cd, mtd);
 					fmt = FileFormat.safeValueOf(mtd.getString(DataExpression.FORMAT_TYPE));
 				}