You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2022/09/15 18:08:16 UTC
[systemds] branch main updated: [SYSTEMS-3432] Estimate NNZ/sparsity in bindResponses
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 35f4e79b54 [SYSTEMS-3432] Estimate NNZ/sparsity in bindResponses
35f4e79b54 is described below
commit 35f4e79b5483415c568dbf93aeec9311e86fe87c
Author: Andreas Botzner <an...@botzner.com>
AuthorDate: Mon Aug 29 09:46:22 2022 +0200
[SYSTEMS-3432] Estimate NNZ/sparsity in bindResponses
This commit analyze if the result matrix should be sparse before
allocation when getting a federated response back to the controller,
And allocate a sparse or dense matrix accordingly.
Closes #1692
---
.../runtime/controlprogram/federated/FederationUtils.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java
index 269d91e9c5..02aefb928c 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java
@@ -145,7 +145,7 @@ public class FederationUtils {
Lop.OPERAND_DELIMITOR + varOldOut.getName() + Lop.DATATYPE_PREFIX,
Lop.OPERAND_DELIMITOR + String.valueOf(outputId) + Lop.DATATYPE_PREFIX);
}
-
+
fr[j] = new FederatedRequest(RequestType.EXEC_INST, outputId, (Object) linst[j]);
}
return fr;
@@ -539,7 +539,13 @@ public class FederationUtils {
public static MatrixBlock bindResponses(List<Pair<FederatedRange, Future<FederatedResponse>>> readResponses, long[] dims)
throws Exception
{
- MatrixBlock ret = new MatrixBlock((int) dims[0], (int) dims[1], false);
+ long totalNNZ = 0;
+ for(Pair<FederatedRange, Future<FederatedResponse>> readResponse : readResponses) {
+ FederatedResponse response = readResponse.getRight().get();
+ MatrixBlock multRes = (MatrixBlock) response.getData()[0];
+ totalNNZ += multRes.getNonZeros();
+ }
+ MatrixBlock ret = new MatrixBlock((int) dims[0], (int) dims[1], MatrixBlock.evalSparseFormatInMemory(dims[0], dims[1], totalNNZ));
for(Pair<FederatedRange, Future<FederatedResponse>> readResponse : readResponses) {
FederatedRange range = readResponse.getLeft();
FederatedResponse response = readResponse.getRight().get();