You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2022/08/24 13:56:31 UTC
[systemds] branch main updated: [SYSTEMDS-3424] Federated Statistics print in non federated scenario
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new c018950443 [SYSTEMDS-3424] Federated Statistics print in non federated scenario
c018950443 is described below
commit c0189504432cecf09c6629442fab5e04c9edfc8a
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Wed Aug 24 15:55:49 2022 +0200
[SYSTEMDS-3424] Federated Statistics print in non federated scenario
---
.../controlprogram/federated/FederatedRequest.java | 21 ++++++++++++----
.../federated/FederatedStatistics.java | 28 +++++++++++-----------
.../monitoring/services/StatisticsService.java | 27 +++++++++++++--------
.../runtime/instructions/InstructionParser.java | 13 ++++------
4 files changed, 51 insertions(+), 38 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedRequest.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedRequest.java
index 0ae8ac033c..163988f5f6 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedRequest.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedRequest.java
@@ -211,11 +211,22 @@ public class FederatedRequest implements Serializable {
@Override
public String toString() {
StringBuilder sb = new StringBuilder("FederatedRequest[");
- sb.append(_method); sb.append(";");
- sb.append(_pid); sb.append(";");
- sb.append(_id); sb.append(";");
- sb.append("t"); sb.append(_tid); sb.append(";");
- sb.append(_data.toString());
+ sb.append(_method);
+ sb.append(";");
+ sb.append(_pid);
+ sb.append(";");
+ sb.append(_id);
+ sb.append(";t");
+ sb.append(_tid);
+ if(_data.size() > 0) {
+ sb.append(";[");
+ for(Object o : _data) {
+ sb.append(o.getClass().getSimpleName());
+ sb.append(", ");
+ }
+ sb.delete(sb.length() - 2, sb.length());
+ sb.append("]");
+ }
sb.append("]");
return sb.toString();
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java
index 32e85252d5..99a23fa753 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java
@@ -284,20 +284,20 @@ public class FederatedStatistics {
}
public static String displayFedWorkerStats() {
- StringBuilder sb = new StringBuilder();
- sb.append(displayFedLookupTableStats());
- sb.append(displayFedReuseReadStats());
- sb.append(displayFedPutLineageStats());
- sb.append(displayFedSerializationReuseStats());
- sb.append(displayFedTransfer());
- //FIXME: the following statistics need guards to only show
- // results if federated operations where executed, also the CPU
- // and mem usage only probe once at the time of stats printing
- //sb.append(displayFedTransfer());
- //sb.append(displayCPUUsage());
- //sb.append(displayMemoryUsage());
-
- return sb.toString();
+ if( readCount.longValue() > 0){
+ StringBuilder sb = new StringBuilder();
+ sb.append(displayFedLookupTableStats());
+ sb.append(displayFedReuseReadStats());
+ sb.append(displayFedPutLineageStats());
+ sb.append(displayFedSerializationReuseStats());
+
+ //sb.append(displayFedTransfer());
+ //sb.append(displayCPUUsage());
+ //sb.append(displayMemoryUsage());
+
+ return sb.toString();
+ }
+ return "";
}
public static String displayStatistics(int numHeavyHitters) {
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatisticsService.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatisticsService.java
index 0b05effce9..d60f7935a4 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatisticsService.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatisticsService.java
@@ -19,25 +19,32 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.services;
-import org.apache.sysds.api.DMLScript;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.controlprogram.federated.FederatedData;
import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest;
import org.apache.sysds.runtime.controlprogram.federated.FederatedResponse;
import org.apache.sysds.runtime.controlprogram.federated.FederatedStatistics;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.*;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.CoordinatorConnectionModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.CoordinatorModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.DataObjectModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.EventModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.EventStageModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.RequestModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatisticsModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatisticsOptions;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.TrafficModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.UtilizationModel;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.Constants;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.DerbyRepository;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.IRepository;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Future;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
public class StatisticsService {
private static final IRepository entityRepository = new DerbyRepository();
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/InstructionParser.java b/src/main/java/org/apache/sysds/runtime/instructions/InstructionParser.java
index e5c9633d97..ccc97fd15c 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/InstructionParser.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/InstructionParser.java
@@ -26,7 +26,6 @@ import org.apache.sysds.runtime.instructions.fed.FEDInstruction;
import org.apache.sysds.runtime.instructions.gpu.GPUInstruction.GPUINSTRUCTION_TYPE;
import org.apache.sysds.runtime.instructions.spark.SPInstruction.SPType;
-
public class InstructionParser
{
public static Instruction parseSingleInstruction ( String str ) {
@@ -36,30 +35,26 @@ public class InstructionParser
ExecType et = InstructionUtils.getExecType(str);
switch( et ) {
case CP:
- case CP_FILE: {
+ case CP_FILE:
CPType cptype = InstructionUtils.getCPType(str);
if( cptype == null )
throw new DMLRuntimeException("Unknown CP instruction: " + str);
return CPInstructionParser.parseSingleInstruction (cptype, str);
- }
- case SPARK: {
+ case SPARK:
SPType sptype = InstructionUtils.getSPType(str);
if( sptype == null )
throw new DMLRuntimeException("Unknown SPARK instruction: " + str);
return SPInstructionParser.parseSingleInstruction (sptype, str);
- }
- case GPU: {
+ case GPU:
GPUINSTRUCTION_TYPE gputype = InstructionUtils.getGPUType(str);
if( gputype == null )
throw new DMLRuntimeException("Unknown GPU instruction: " + str);
return GPUInstructionParser.parseSingleInstruction (gputype, str);
- }
- case FED: {
+ case FED:
FEDInstruction.FEDType fedtype = InstructionUtils.getFEDType(str);
if( fedtype == null )
throw new DMLRuntimeException("Unknown FEDERATED instruction: " + str);
return FEDInstructionParser.parseSingleInstruction (fedtype, str);
- }
default:
throw new DMLRuntimeException("Unknown execution type in instruction: " + str);
}