You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2022/08/18 13:40:02 UTC
[systemds] branch main updated: [MINOR] Fix syntax warnings introduced in SYSTEMDS-3385
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new a411e80f40 [MINOR] Fix syntax warnings introduced in SYSTEMDS-3385
a411e80f40 is described below
commit a411e80f40975720ec20688c0acd699b63d8bce3
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Thu Aug 18 15:39:48 2022 +0200
[MINOR] Fix syntax warnings introduced in SYSTEMDS-3385
---
pom.xml | 1 +
src/main/java/org/apache/sysds/api/DMLScript.java | 15 +++++---
.../controlprogram/federated/FederatedData.java | 28 ++++-----------
.../federated/FederatedStatistics.java | 4 ---
.../monitoring/FederatedMonitoringServer.java | 4 +--
.../controlprogram/federated/monitoring/README.md | 40 ++++++++++++++++++----
.../monitoring/controllers/WorkerController.java | 6 ++--
.../models/CoordinatorConnectionModel.java | 3 --
.../monitoring/models/DataObjectModel.java | 3 --
.../federated/monitoring/models/EventModel.java | 1 -
.../monitoring/models/EventStageModel.java | 1 -
.../federated/monitoring/models/TrafficModel.java | 5 ---
.../monitoring/models/UtilizationModel.java | 1 -
.../federated/monitoring/models/WorkerModel.java | 2 --
.../monitoring/FederatedMonitoringTestBase.java | 13 +++----
.../monitoring/FederatedWorkerStatisticsTest.java | 10 +++---
16 files changed, 68 insertions(+), 69 deletions(-)
diff --git a/pom.xml b/pom.xml
index 7144a4789b..1ace5488d7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,6 +54,7 @@
<log4j.version>2.17.2</log4j.version>
<!-- Set java compile level via argument, ex: 1.8 1.9 10 11-->
<java.level>11</java.level>
+ <java.version>{java.level}</java.version>
<!-->Testing settings<!-->
<maven.test.skip>true</maven.test.skip>
<test-parallel>classes</test-parallel>
diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java
index 9f0ab9dfb7..881cd47040 100644
--- a/src/main/java/org/apache/sysds/api/DMLScript.java
+++ b/src/main/java/org/apache/sysds/api/DMLScript.java
@@ -408,8 +408,8 @@ public class DMLScript
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs)
throws IOException
{
- //print basic time environment info and process id
- printStartExecInfo( dmlScriptStr );
+ // print basic time, environment info, and process id
+ printStartExecInfo(dmlScriptStr);
//Step 1: parse configuration files & write any configuration specific global variables
loadConfiguration(fnameOptConfig);
@@ -576,9 +576,14 @@ public class DMLScript
}
private static void printStartExecInfo(String dmlScriptString) {
- LOG.info("BEGIN DML run " + getDateTime());
- LOG.debug("DML script: \n" + dmlScriptString);
- LOG.info("Process id: " + IDHandler.obtainProcessID());
+ boolean info = LOG.isInfoEnabled();
+ boolean debug = LOG.isDebugEnabled();
+ if(info)
+ LOG.info("BEGIN DML run " + getDateTime());
+ if(debug)
+ LOG.debug("DML script: \n" + dmlScriptString);
+ if(info)
+ LOG.info("Process id: " + IDHandler.obtainProcessID());
}
private static String getDateTime() {
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
index 9497de1f2f..7851912d76 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
@@ -19,11 +19,8 @@
package org.apache.sysds.runtime.controlprogram.federated;
-import java.io.IOException;
import java.io.Serializable;
import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -32,21 +29,25 @@ import java.util.concurrent.Future;
import javax.net.ssl.SSLException;
-import io.netty.channel.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.sysds.api.DMLScript;
import org.apache.sysds.common.Types;
import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.conf.DMLConfig;
+import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.controlprogram.caching.CacheBlock;
import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType;
import org.apache.sysds.runtime.controlprogram.paramserv.NetworkTrafficCounter;
-import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.meta.MetaData;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
@@ -71,7 +72,6 @@ public class FederatedData {
private final Types.DataType _dataType;
private final InetSocketAddress _address;
private final String _filepath;
- private static final int endOfDynamicPorts = 65535;
/**
* The ID of default matrix/tensor on which operations get executed if no other ID is given.
@@ -200,20 +200,6 @@ public class FederatedData {
}
}
- private static int getAvailablePort(int monitorId, int maxMonitorCoordinators) {
-
- for (int i = 0; i < maxMonitorCoordinators; i++) {
- int tmpPort = endOfDynamicPorts - monitorId - i * maxMonitorCoordinators;
- try(ServerSocket availableSocket = new ServerSocket(tmpPort)) {
- return availableSocket.getLocalPort();
- }
- catch(IOException ignored) {
- }
- }
-
- return -1;
- }
-
private static ChannelInitializer<SocketChannel> createChannel(InetSocketAddress address, DataRequestHandler handler){
final int timeout = ConfigurationManager.getFederatedTimeout();
final boolean ssl = ConfigurationManager.isFederatedSSL();
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java
index 46d6d26e13..32e85252d5 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java
@@ -21,15 +21,12 @@ package org.apache.sysds.runtime.controlprogram.federated;
import java.io.Serializable;
import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryMXBean;
-import java.lang.management.ThreadMXBean;
import java.net.InetSocketAddress;
import java.text.DecimalFormat;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
-import java.util.concurrent.CopyOnWriteArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -42,7 +39,6 @@ import java.util.concurrent.atomic.LongAdder;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.sysds.api.DMLScript;
-import org.apache.sysds.common.Types;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.controlprogram.caching.CacheBlock;
import org.apache.sysds.runtime.controlprogram.caching.CacheStatistics;
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java
index 031866c279..6b3b180a26 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java
@@ -19,6 +19,8 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring;
+import org.apache.log4j.Logger;
+
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
@@ -30,10 +32,8 @@ import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpServerCodec;
-import io.netty.handler.codec.http.cors.CorsConfig;
import io.netty.handler.codec.http.cors.CorsConfigBuilder;
import io.netty.handler.codec.http.cors.CorsHandler;
-import org.apache.log4j.Logger;
public class FederatedMonitoringServer {
protected static Logger log = Logger.getLogger(FederatedMonitoringServer.class);
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/README.md b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/README.md
index 1a82b1d804..65377f675f 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/README.md
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/README.md
@@ -1,9 +1,26 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% end comment %}
+-->
# Backend for monitoring tool of federated infrastructure
A backend application, used to collect, store, aggregate and return metrics data from coordinators and workers in the cluster
-
## Install & Run
The backend process can be started in a similar manner with how a worker is started:
@@ -19,29 +36,34 @@ Or with the specified **-fedMonitor 8080** flag indicating the start of the back
## Main components
### Architecture
-The following diagram illustrates the processes running in the backend.
+The following diagram illustrates the processes running in the backend.
#### Controller
+
Serves as the main integration point between the frontend and backend.
#### Service
+
Holds the business logic of the backend application.
#### Repository
+
serves as the main integration point between the backend and the chosen persistent storage. It can be extended to persist data in the file system, by extending the **IRepository** class and changing the instance in the service classes.
### Database schema
-The following diagram illustrates the current state of the database schema.
+The following diagram illustrates the current state of the database schema.
-**Important to note**
-- There is no foreign key constraint between the worker and statistics tables.
+### Important to note
+
+- There is no foreign key constraint between the worker and statistics tables.
- The field for **coordinatorTraffic** is parsed into JSON format upon retrieval and saved as a string in the database. Example:
+
```json
{
"datetime": "2022-06-24T17:08:56.897188",
@@ -49,7 +71,9 @@ The following diagram illustrates the current state of the database schema.
"byteAmount": 45000
}
```
+
- The field for **heavyHitters** is parsed into JSON format upon retrieval and saved as a string in the database. Example:
+
```json
{
"instruction": "fed_uamin",
@@ -59,13 +83,15 @@ The following diagram illustrates the current state of the database schema.
```
### Processes
-The following diagram illustrates the processes running in the backend.
+The following diagram illustrates the processes running in the backend.
#### Statistics collection thread
+
There is a dedicated thread for the communication between the backend and the workers and statistics are gathered periodically (every 3 seconds by default).
#### Request processing
-The main logic of the application listens for REST requests coming from the frontend.
\ No newline at end of file
+
+The main logic of the application listens for REST requests coming from the frontend.
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java
index 52e318182e..95e47c54b1 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java
@@ -19,14 +19,14 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers;
-import io.netty.handler.codec.http.FullHttpResponse;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.Request;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.Response;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.MapperService;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.StatisticsService;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.WorkerService;
+import io.netty.handler.codec.http.FullHttpResponse;
+
public class WorkerController implements IController {
private final WorkerService workerService = new WorkerService();
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java
index 5999f59a11..dc28232225 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java
@@ -19,9 +19,6 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import java.io.Serializable;
-import java.time.LocalDateTime;
-
public abstract class CoordinatorConnectionModel extends BaseModel {
public Long coordinatorId;
private String coordinatorHostId;
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java
index b8c218c749..5e20ba2887 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java
@@ -19,9 +19,6 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import java.io.Serializable;
-import java.time.LocalDateTime;
-
public class DataObjectModel extends BaseModel {
public Long workerId;
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java
index b8e3bd85cb..fc4694c935 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java
@@ -19,7 +19,6 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java
index cd26bfbdb4..1171a7c25c 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java
@@ -19,7 +19,6 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import java.io.Serializable;
import java.time.LocalDateTime;
public class EventStageModel extends BaseModel {
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java
index d4b4b860ea..9d0d1d160e 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java
@@ -19,7 +19,6 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import java.io.Serializable;
import java.time.LocalDateTime;
public class TrafficModel extends CoordinatorConnectionModel {
@@ -43,10 +42,6 @@ public class TrafficModel extends CoordinatorConnectionModel {
super.setCoordinatorHostId(coordinatorHostId);
}
- private TrafficModel(final Long id) {
- this.id = id;
- }
-
public TrafficModel(final Long workerId, final String coordinatorAddress, final Long byteAmount) {
this(-1L, workerId, LocalDateTime.now(), coordinatorAddress, byteAmount);
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java
index 83a5d290e4..d92555a3ee 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java
@@ -19,7 +19,6 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import java.io.Serializable;
import java.time.LocalDateTime;
public class UtilizationModel extends BaseModel {
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java
index bd26e5235d..922fe9239a 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java
@@ -19,8 +19,6 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import java.util.List;
-
public class WorkerModel extends BaseModel {
public String name;
public String address;
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedMonitoringTestBase.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedMonitoringTestBase.java
index 5d611d6388..2a0901b15a 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedMonitoringTestBase.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedMonitoringTestBase.java
@@ -19,11 +19,6 @@
package org.apache.sysds.test.functions.federated.monitoring;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
-import org.apache.sysds.test.functions.federated.multitenant.MultiTenantTestBase;
-import org.junit.After;
-
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
@@ -32,6 +27,12 @@ import java.net.http.HttpResponse;
import java.util.ArrayList;
import java.util.List;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
+import org.apache.sysds.test.functions.federated.multitenant.MultiTenantTestBase;
+import org.junit.After;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
public abstract class FederatedMonitoringTestBase extends MultiTenantTestBase {
protected Process monitoringProcess;
private int monitoringPort;
@@ -39,7 +40,7 @@ public abstract class FederatedMonitoringTestBase extends MultiTenantTestBase {
private static final String MAIN_URI = "http://localhost";
private static final String WORKER_MAIN_PATH = "/workers";
- private static final String COORDINATOR_MAIN_PATH = "/coordinators";
+ // private static final String COORDINATOR_MAIN_PATH = "/coordinators";
@Override
public abstract void setUp();
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerStatisticsTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerStatisticsTest.java
index af39724f70..5d092daf8e 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerStatisticsTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerStatisticsTest.java
@@ -72,14 +72,14 @@ public class FederatedWorkerStatisticsTest extends FederatedMonitoringTestBase {
var bla = new EventModel(1L, -1L);
var derby = new DerbyRepository();
- var in1 = derby.createEntity(bla);
- var in2 = derby.createEntity(bla);
+ derby.createEntity(bla);
+ derby.createEntity(bla);
var in3 = derby.createEntity(bla);
- var in4 = derby.createEntity(bla);
+ derby.createEntity(bla);
- var shit = derby.getEntity(in3, EventModel.class);
+ derby.getEntity(in3, EventModel.class);
- var stage = new EventStageModel();
+ new EventStageModel();
workerMonitoringService.create(new WorkerModel(1L, "Worker", "localhost:8001"));