You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ja...@apache.org on 2022/08/18 01:50:56 UTC
[systemds] branch main updated: [SYSTEMDS-3385] Add and integrate Federated monitoring frontend UI
This is an automated email from the ASF dual-hosted git repository.
janardhan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 7286241964 [SYSTEMDS-3385] Add and integrate Federated monitoring frontend UI
7286241964 is described below
commit 728624196442e59991789ae677241e9a1dfd3953
Author: Mito Kehayov <mk...@arakt.com>
AuthorDate: Wed Aug 17 17:57:56 2022 +0530
[SYSTEMDS-3385] Add and integrate Federated monitoring frontend UI
Design Goals:
* A Robust Object Relational Model (ORM) with Model View
Controller (MVC)
* Worker and Coordinator details, statistics and Event timeline
* Extendability of new statistics and features - with Angular framework
we can achieve this goal.
* Testing. Documentation with architecture diagrams
Functionality:
* Frontend integration of entities CRUD + real-time metrics
monitoring
* Dashboard traffic and worker component visualization
* Event timeline
Future direction:
* Where possible, use this monitoring UI. and note any improvements
to extend this feature based on those needs.
* Use a helm chart or docker compose to simplify the use of this tool.
Closes #1641.
---
pom.xml | 1 +
scripts/monitoring/.browserslistrc | 33 ++
scripts/monitoring/.editorconfig | 34 +++
scripts/monitoring/.gitignore | 43 +++
scripts/monitoring/README.md | 172 +++++++++++
scripts/monitoring/angular.json | 113 +++++++
scripts/monitoring/karma.conf.js | 64 ++++
scripts/monitoring/package.json | 47 +++
scripts/monitoring/src/app/app-routing.module.ts | 69 +++++
scripts/monitoring/src/app/app.component.html | 20 ++
.../monitoring/src/app/app.component.scss | 4 -
.../monitoring/src/app/app.component.spec.ts | 47 +--
.../monitoring/src/app/app.component.ts | 13 +-
scripts/monitoring/src/app/app.module.ts | 80 +++++
.../monitoring/src/app/constants.ts | 40 ++-
scripts/monitoring/src/app/material.module.ts | 143 +++++++++
.../monitoring/src/app/models/coordinator.model.ts | 11 +-
.../monitoring/src/app/models/dataObject.model.ts | 11 +-
.../monitoring/src/app/models/event.model.ts | 9 +-
.../monitoring/src/app/models/eventStage.model.ts | 8 +-
.../monitoring/src/app/models/fedRequest.model.ts | 7 +-
.../monitoring/src/app/models/fedSiteData.model.ts | 10 +-
.../monitoring/src/app/models/statistics.model.ts | 16 +-
.../monitoring/src/app/models/traffic.model.ts | 10 +-
.../monitoring/src/app/models/utilization.model.ts | 10 +-
.../monitoring/src/app/models/worker.model.ts | 10 +-
.../create-edit/create-edit.component.html | 43 +++
.../create-edit/create-edit.component.scss | 6 +-
.../create-edit/create-edit.component.spec.ts | 68 +++++
.../create-edit/create-edit.component.ts | 62 ++++
.../modules/coordinators/list/list.component.html | 71 +++++
.../modules/coordinators/list/list.component.scss | 43 ++-
.../coordinators/list/list.component.spec.ts | 72 +++++
.../modules/coordinators/list/list.component.ts | 75 +++++
.../dashboard/connection/connection.component.html | 22 ++
.../dashboard/connection/connection.component.scss | 7 +-
.../connection/connection.component.spec.ts | 44 +--
.../dashboard/connection/connection.component.ts | 109 +++++++
.../coordinator/coordinator.component.html | 40 +++
.../coordinator/coordinator.component.scss | 20 +-
.../coordinator/coordinator.component.spec.ts | 48 +--
.../dashboard/coordinator/coordinator.component.ts | 21 +-
.../dialog-dashboard.component.html | 46 +++
.../dialog-dashboard.component.scss | 15 +-
.../dialog-dashboard.component.spec.ts | 60 ++++
.../dialog-dashboard/dialog-dashboard.component.ts | 68 +++++
.../dashboard/main/dashboard.component.html | 24 ++
.../dashboard/main/dashboard.component.scss | 20 +-
.../dashboard/main/dashboard.component.spec.ts | 60 ++++
.../modules/dashboard/main/dashboard.component.ts | 140 +++++++++
.../modules/dashboard/main/dashboard.directive.ts | 12 +-
.../modules/dashboard/worker/worker.component.html | 78 +++++
.../modules/dashboard/worker/worker.component.scss | 35 ++-
.../dashboard/worker/worker.component.spec.ts | 66 ++++
.../modules/dashboard/worker/worker.component.ts | 147 +++++++++
.../app/modules/events/list/list.component.html | 75 +++++
.../app/modules/events/list/list.component.scss | 54 +++-
.../app/modules/events/list/list.component.spec.ts | 72 +++++
.../src/app/modules/events/list/list.component.ts | 66 ++++
.../app/modules/events/view/view.component.html | 25 ++
.../app/modules/events/view/view.component.scss | 106 +++++++
.../app/modules/events/view/view.component.spec.ts | 82 +++++
.../src/app/modules/events/view/view.component.ts | 281 +++++++++++++++++
.../src/app/modules/layout/layout.component.html | 113 +++++++
.../src/app/modules/layout/layout.component.scss | 83 +++++
.../app/modules/layout/layout.component.spec.ts | 79 +++++
.../src/app/modules/layout/layout.component.ts | 72 +++++
.../workers/create-edit/create-edit.component.html | 38 +++
.../workers/create-edit/create-edit.component.scss | 6 +-
.../create-edit/create-edit.component.spec.ts | 67 ++++
.../workers/create-edit/create-edit.component.ts | 66 ++++
.../app/modules/workers/list/list.component.html | 80 +++++
.../app/modules/workers/list/list.component.scss | 54 +++-
.../modules/workers/list/list.component.spec.ts | 73 +++++
.../src/app/modules/workers/list/list.component.ts | 80 +++++
.../app/modules/workers/view/view.component.html | 89 ++++++
.../app/modules/workers/view/view.component.scss | 110 +++++++
.../modules/workers/view/view.component.spec.ts | 86 ++++++
.../src/app/modules/workers/view/view.component.ts | 213 +++++++++++++
.../app/services/federatedSiteService.service.ts | 104 +++++++
.../src/app/services/federatedSiteService.stub.ts | 75 +++++
.../src/app/services/service-mock-data.ts | 108 +++++++
.../monitoring/src/app/utils.ts | 12 +-
scripts/monitoring/src/assets/favicon.png | Bin 0 -> 461 bytes
.../src/environments/environment.prod.ts | 6 +-
.../monitoring/src/environments/environment.ts | 31 +-
scripts/monitoring/src/index.html | 35 +++
.../monitoring/src/main.ts | 15 +-
scripts/monitoring/src/polyfills.ts | 72 +++++
.../monitoring/src/styles.scss | 13 +-
.../monitoring/src/test.ts | 43 +--
.../monitoring/tsconfig.app.json | 20 +-
scripts/monitoring/tsconfig.json | 55 ++++
.../monitoring/tsconfig.spec.json | 23 +-
src/main/java/org/apache/sysds/api/DMLOptions.java | 5 +
src/main/java/org/apache/sysds/api/DMLScript.java | 5 +-
.../controlprogram/federated/FederatedData.java | 26 +-
.../federated/FederatedStatistics.java | 154 ++++++----
.../federated/FederatedWorkerHandler.java | 119 ++++++--
.../federated/monitoring/Backend-architecture.svg | 4 +
.../federated/monitoring/Backend-processes.svg | 4 +
.../federated/monitoring/DB-diagram.svg | 4 +
.../monitoring/FederatedMonitoringServer.java | 18 ++
.../FederatedMonitoringServerHandler.java | 74 ++---
.../controlprogram/federated/monitoring/README.md | 71 +++++
.../federated/monitoring/{models => }/Request.java | 2 +-
.../monitoring/{models => }/Response.java | 14 +-
.../EntityEnum.java => controllers/Constants.java} | 10 +-
.../controllers/CoordinatorController.java | 35 ++-
.../monitoring/controllers/IController.java | 2 +-
...erController.java => StatisticsController.java} | 46 +--
.../monitoring/controllers/WorkerController.java | 38 +--
.../{BaseEntityModel.java => BaseModel.java} | 6 +-
...equest.java => CoordinatorConnectionModel.java} | 33 +-
.../models/{Request.java => CoordinatorModel.java} | 34 ++-
.../monitoring/models/DataObjectModel.java | 64 ++++
.../federated/monitoring/models/EventModel.java | 70 +++++
.../models/{Request.java => EventStageModel.java} | 41 ++-
.../monitoring/models/NodeEntityModel.java | 80 -----
.../models/{Request.java => RequestModel.java} | 36 ++-
.../monitoring/models/StatisticsModel.java | 91 ++++++
...BaseEntityModel.java => StatisticsOptions.java} | 9 +-
.../monitoring/models/StatsEntityModel.java | 139 ---------
.../federated/monitoring/models/TrafficModel.java | 67 ++++
.../monitoring/models/UtilizationModel.java | 66 ++++
.../IController.java => models/WorkerModel.java} | 41 ++-
.../monitoring/repositories/Constants.java | 16 +-
.../monitoring/repositories/DerbyRepository.java | 336 +++++++++++++--------
.../monitoring/repositories/IRepository.java | 20 +-
.../monitoring/services/CoordinatorService.java | 23 +-
.../monitoring/services/MapperService.java | 75 ++---
.../monitoring/services/StatisticsService.java | 184 +++++++++++
.../monitoring/services/StatsService.java | 78 -----
.../monitoring/services/WorkerService.java | 144 +++++----
.../FederatedCoordinatorIntegrationCRUDTest.java | 25 +-
.../monitoring/FederatedMonitoringTestBase.java | 32 +-
.../FederatedWorkerIntegrationCRUDTest.java | 25 +-
.../monitoring/FederatedWorkerStatisticsTest.java | 44 ++-
138 files changed, 6495 insertions(+), 1144 deletions(-)
diff --git a/pom.xml b/pom.xml
index 6268be7b34..7144a4789b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -95,6 +95,7 @@
<exclude>perftestDeprecated/*</exclude>
<exclude>perftestDeprecated</exclude>
<exclude>staging/**/*</exclude>
+ <exclude>monitoring/**/*</exclude>
<exclude>nn/test/compare_backends/*</exclude>
<exclude>nn/test/compare_backends/*</exclude>
</excludes>
diff --git a/scripts/monitoring/.browserslistrc b/scripts/monitoring/.browserslistrc
new file mode 100644
index 0000000000..fda4b9bc17
--- /dev/null
+++ b/scripts/monitoring/.browserslistrc
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This file is used by the build system to adjust CSS and JS output to support the specified browsers below.
+# For additional information regarding the format and rule options, please see:
+# https://github.com/browserslist/browserslist#queries
+
+# For the full list of supported browsers by the Angular framework, please see:
+# https://angular.io/guide/browser-support
+
+# You can see what browsers were selected by your queries by running:
+# npx browserslist
+
+last 1 Chrome version
+last 1 Firefox version
+last 2 Edge major versions
+last 2 Safari major versions
+last 2 iOS major versions
+Firefox ESR
diff --git a/scripts/monitoring/.editorconfig b/scripts/monitoring/.editorconfig
new file mode 100644
index 0000000000..c188b80919
--- /dev/null
+++ b/scripts/monitoring/.editorconfig
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+ij_typescript_spaces_within_imports = true
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false
diff --git a/scripts/monitoring/.gitignore b/scripts/monitoring/.gitignore
new file mode 100644
index 0000000000..1a180e8aa0
--- /dev/null
+++ b/scripts/monitoring/.gitignore
@@ -0,0 +1,43 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+
+# Compiled output
+dist
+tmp
+out-tsc
+bazel-out
+
+# Node
+node_modules
+npm-debug.log
+yarn-error.log
+
+# IDEs and editors
+.idea/
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# Visual Studio Code
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history/*
+
+# Miscellaneous
+.angular/cache
+.sass-cache/
+connect.lock
+coverage
+libpeerconnection.log
+testem.log
+typings
+package-lock.json
+
+# System files
+.DS_Store
+Thumbs.db
diff --git a/scripts/monitoring/README.md b/scripts/monitoring/README.md
new file mode 100644
index 0000000000..6a5fe74d66
--- /dev/null
+++ b/scripts/monitoring/README.md
@@ -0,0 +1,172 @@
+
+# Frontend for monitoring tool of federated infrastrucuture
+
+A frontend application, used to visualize and manipulate the backend application functionality of the monitoring tool
+
+
+## Backend requirements
+
+A running instance of the backend application is required for the frontend to function, default port of the backend is **8080**.
+
+
+## Install & Run
+
+To install and run the app do the following:
+
+```bash
+ cd scripts/monitoring
+ npm install
+ npm run start
+```
+To view the app Navigate to `http://localhost:4200/`.
+## Running Tests
+
+To run tests, run the following command:
+
+```bash
+ npm run test
+```
+
+
+## API Reference
+
+#### Get all registered workers
+
+```http
+ GET /workers
+```
+
+#### Get specific worker
+
+```http
+ GET /workers/${id}
+```
+
+| Parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `id` | `int` | **Required**. Id of the worker to fetch |
+
+#### Register worker for monitoring
+```http
+ POST /workers
+```
+##### Request body in **JSON** format:
+
+| Body parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `name` | `string` | **Required**. Name of the worker to register |
+| `address` | `string` | **Required**. Address of the worker to register |
+
+##### Example:
+
+```json
+{
+ "name": "Worker 1",
+ "address": "localhost:8001"
+}
+```
+#### Edit registered worker
+```http
+ PUT /workers/${id}
+```
+| Parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `id` | `int` | **Required**. Id of the worker to edit |
+
+##### Request body in **JSON** format:
+
+| Body parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `name` | `string` | Changed name of the worker |
+| `address` | `string` | Changed address of the worker |
+
+##### Example:
+
+```json
+{
+ "name": "Worker 42",
+ "address": "localhost:8005"
+}
+```
+#### Deregister specific worker
+
+```http
+ DELETE /workers/${id}
+```
+
+| Parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `id` | `int` | **Required**. Id of the worker to deregister |
+
+---
+#### Get all registered coordinators
+
+```http
+ GET /coordinators
+```
+
+#### Get specific coordinator
+
+```http
+ GET /coordinators/${id}
+```
+
+| Parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `id` | `int` | **Required**. Id of the coordinator to fetch |
+
+#### Register coordinator for monitoring
+```http
+ POST /coordinators
+```
+##### Request body in **JSON** format:
+
+| Body parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `name` | `string` | **Required**. Name of the coordinator to register |
+| `address` | `string` | **Required**. Address of the coordinator to register |
+
+##### Example:
+
+```json
+{
+ "name": "Coordinator 1",
+ "address": "localhost:8441"
+}
+```
+#### Edit registered coordinator
+```http
+ PUT /coordinators/${id}
+```
+| Parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `id` | `int` | **Required**. Id of the coordinator to edit |
+
+##### Request body in **JSON** format:
+
+| Body parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `name` | `string` | Changed name of the coordinator |
+| `address` | `string` | Changed address of the coordinator |
+
+##### Example:
+
+```json
+{
+ "name": "Coordinator 4",
+ "address": "localhost:8445"
+}
+```
+#### Deregister specific coordinator
+
+```http
+ DELETE /coordinators/${id}
+```
+
+| Parameter | Type | Description |
+| :-------- | :------- | :-------------------------------- |
+| `id` | `int` | **Required**. Id of the coordinator to deregister |
+
+## License
+
+[Apache-2.0](http://www.apache.org/licenses/LICENSE-2.0)
diff --git a/scripts/monitoring/angular.json b/scripts/monitoring/angular.json
new file mode 100644
index 0000000000..cde23070c4
--- /dev/null
+++ b/scripts/monitoring/angular.json
@@ -0,0 +1,113 @@
+{
+ "$schema": "./node_modules/@angular/cli/lib/config/schema.json",
+ "version": 1,
+ "newProjectRoot": "projects",
+ "projects": {
+ "monitoring-ui": {
+ "projectType": "application",
+ "schematics": {
+ "@schematics/angular:component": {
+ "style": "scss"
+ },
+ "@schematics/angular:application": {
+ "strict": true
+ }
+ },
+ "root": "",
+ "sourceRoot": "src",
+ "prefix": "app",
+ "architect": {
+ "build": {
+ "builder": "@angular-devkit/build-angular:browser",
+ "options": {
+ "outputPath": "dist/monitoring-ui",
+ "index": "src/index.html",
+ "main": "src/main.ts",
+ "polyfills": "src/polyfills.ts",
+ "tsConfig": "tsconfig.app.json",
+ "inlineStyleLanguage": "scss",
+ "assets": [
+ "src/favicon.ico",
+ "src/assets"
+ ],
+ "styles": [
+ "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css",
+ "src/styles.scss"
+ ],
+ "scripts": []
+ },
+ "configurations": {
+ "production": {
+ "budgets": [
+ {
+ "type": "initial",
+ "maximumWarning": "1mb",
+ "maximumError": "5mb"
+ },
+ {
+ "type": "anyComponentStyle",
+ "maximumWarning": "2kb",
+ "maximumError": "4kb"
+ }
+ ],
+ "fileReplacements": [
+ {
+ "replace": "src/environments/environment.ts",
+ "with": "src/environments/environment.prod.ts"
+ }
+ ],
+ "outputHashing": "all"
+ },
+ "development": {
+ "buildOptimizer": false,
+ "optimization": false,
+ "vendorChunk": true,
+ "extractLicenses": false,
+ "sourceMap": true,
+ "namedChunks": true
+ }
+ },
+ "defaultConfiguration": "production"
+ },
+ "serve": {
+ "builder": "@angular-devkit/build-angular:dev-server",
+ "configurations": {
+ "production": {
+ "browserTarget": "monitoring-ui:build:production"
+ },
+ "development": {
+ "browserTarget": "monitoring-ui:build:development"
+ }
+ },
+ "defaultConfiguration": "development"
+ },
+ "extract-i18n": {
+ "builder": "@angular-devkit/build-angular:extract-i18n",
+ "options": {
+ "browserTarget": "monitoring-ui:build"
+ }
+ },
+ "test": {
+ "builder": "@angular-devkit/build-angular:karma",
+ "options": {
+ "main": "src/test.ts",
+ "polyfills": "src/polyfills.ts",
+ "tsConfig": "tsconfig.spec.json",
+ "karmaConfig": "karma.conf.js",
+ "inlineStyleLanguage": "scss",
+ "assets": [
+ "src/favicon.ico",
+ "src/assets"
+ ],
+ "styles": [
+ "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css",
+ "src/styles.scss"
+ ],
+ "scripts": []
+ }
+ }
+ }
+ }
+ },
+ "defaultProject": "monitoring-ui"
+}
diff --git a/scripts/monitoring/karma.conf.js b/scripts/monitoring/karma.conf.js
new file mode 100644
index 0000000000..99251b9edf
--- /dev/null
+++ b/scripts/monitoring/karma.conf.js
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Karma configuration file, see link for more information
+// https://karma-runner.github.io/1.0/config/configuration-file.html
+
+module.exports = function (config) {
+ config.set({
+ basePath: '',
+ frameworks: ['jasmine', '@angular-devkit/build-angular'],
+ plugins: [
+ require('karma-jasmine'),
+ require('karma-chrome-launcher'),
+ require('karma-firefox-launcher'),
+ require('karma-jasmine-html-reporter'),
+ require('karma-coverage'),
+ require('@angular-devkit/build-angular/plugins/karma')
+ ],
+ client: {
+ jasmine: {
+ // you can add configuration options for Jasmine here
+ // the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html
+ // for example, you can disable the random execution with `random: false`
+ // or set a specific seed with `seed: 4321`
+ },
+ clearContext: false // leave Jasmine Spec Runner output visible in browser
+ },
+ jasmineHtmlReporter: {
+ suppressAll: true // removes the duplicated traces
+ },
+ coverageReporter: {
+ dir: require('path').join(__dirname, './coverage/monitoring-ui'),
+ subdir: '.',
+ reporters: [
+ {type: 'html'},
+ {type: 'text-summary'}
+ ]
+ },
+ reporters: ['progress', 'kjhtml'],
+ port: 9876,
+ colors: true,
+ logLevel: config.LOG_INFO,
+ autoWatch: true,
+ browsers: ['Firefox'],
+ singleRun: false,
+ restartOnFileChange: true
+ });
+};
diff --git a/scripts/monitoring/package.json b/scripts/monitoring/package.json
new file mode 100644
index 0000000000..86ca46773c
--- /dev/null
+++ b/scripts/monitoring/package.json
@@ -0,0 +1,47 @@
+{
+ "name": "monitoring-ui",
+ "version": "0.0.0",
+ "scripts": {
+ "ng": "ng",
+ "start": "ng serve",
+ "build": "ng build",
+ "watch": "ng build --watch --configuration development",
+ "test": "ng test"
+ },
+ "private": true,
+ "dependencies": {
+ "@angular/animations": "~13.2.0",
+ "@angular/cdk": "^13.2.6",
+ "@angular/common": "~13.2.0",
+ "@angular/compiler": "~13.2.0",
+ "@angular/core": "~13.2.0",
+ "@angular/forms": "~13.2.0",
+ "@angular/material": "^13.2.6",
+ "@angular/platform-browser": "~13.2.0",
+ "@angular/platform-browser-dynamic": "~13.2.0",
+ "@angular/router": "~13.2.0",
+ "jsplumb": "~2.15.6",
+ "chart.js": "~3.8.2",
+ "chartjs-adapter-moment": "^1.0.0",
+ "moment": "^2.29.4",
+ "rxjs": "~7.5.0",
+ "tslib": "^2.3.0",
+ "zone.js": "~0.11.4"
+ },
+ "devDependencies": {
+ "@angular-devkit/build-angular": "~13.2.6",
+ "@angular/cli": "~13.2.6",
+ "@angular/compiler-cli": "~13.2.0",
+ "@types/jasmine": "~3.10.0",
+ "@types/jest": "^28.1.1",
+ "@types/node": "^12.11.1",
+ "jasmine-core": "~4.0.0",
+ "karma": "~6.3.0",
+ "karma-chrome-launcher": "~3.1.0",
+ "karma-coverage": "~2.1.0",
+ "karma-firefox-launcher": "~2.1.2",
+ "karma-jasmine": "~4.0.0",
+ "karma-jasmine-html-reporter": "~1.7.0",
+ "typescript": "~4.5.2"
+ }
+}
diff --git a/scripts/monitoring/src/app/app-routing.module.ts b/scripts/monitoring/src/app/app-routing.module.ts
new file mode 100644
index 0000000000..1039948503
--- /dev/null
+++ b/scripts/monitoring/src/app/app-routing.module.ts
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { NgModule } from "@angular/core";
+import { RouterModule, Routes } from "@angular/router";
+import { ListCoordinatorsComponent } from "./modules/coordinators/list/list.component";
+import { DashboardComponent } from "./modules/dashboard/main/dashboard.component";
+import { LayoutComponent } from "./modules/layout/layout.component";
+import { ListWorkersComponent } from "./modules/workers/list/list.component";
+import { ViewWorkerComponent } from "./modules/workers/view/view.component";
+import { ListWorkersEventsComponent } from "./modules/events/list/list.component";
+import { ViewWorkerEventsComponent } from "./modules/events/view/view.component";
+
+const routes: Routes = [
+ {
+ path: '',
+ component: LayoutComponent,
+ children: [
+ {
+ path: 'dashboard',
+ component: DashboardComponent
+ },
+ {
+ path: 'coordinators',
+ component: ListCoordinatorsComponent,
+ },
+ {
+ path: 'workers',
+ component: ListWorkersComponent,
+ },
+ {
+ path: 'workers/:id',
+ component: ViewWorkerComponent
+ },
+ {
+ path: 'events',
+ component: ListWorkersEventsComponent,
+ },
+ {
+ path: 'events/:id',
+ component: ViewWorkerEventsComponent
+ },
+ {path: '', redirectTo: 'dashboard', pathMatch: 'full'}
+ ]
+ },
+];
+
+@NgModule({
+ imports: [RouterModule.forRoot(routes, {useHash: true})],
+ exports: [RouterModule]
+})
+export class AppRoutingModule {
+}
diff --git a/scripts/monitoring/src/app/app.component.html b/scripts/monitoring/src/app/app.component.html
new file mode 100644
index 0000000000..36c57afd71
--- /dev/null
+++ b/scripts/monitoring/src/app/app.component.html
@@ -0,0 +1,20 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<router-outlet></router-outlet>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java b/scripts/monitoring/src/app/app.component.scss
similarity index 86%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
copy to scripts/monitoring/src/app/app.component.scss
index 41cf507696..042f3ce1f3 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
+++ b/scripts/monitoring/src/app/app.component.scss
@@ -16,7 +16,3 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-
-public abstract class BaseEntityModel { }
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java b/scripts/monitoring/src/app/app.component.spec.ts
similarity index 52%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
copy to scripts/monitoring/src/app/app.component.spec.ts
index dd683080e2..1df09c0e39 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
+++ b/scripts/monitoring/src/app/app.component.spec.ts
@@ -17,22 +17,31 @@
* under the License.
*/
-
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
-
-import java.util.List;
-
-public interface IRepository {
- Long createEntity(EntityEnum type, BaseEntityModel model);
-
- BaseEntityModel getEntity(EntityEnum type, Long id);
-
- List<BaseEntityModel> getAllEntities(EntityEnum type);
-
- List<BaseEntityModel> getAllEntitiesByField(EntityEnum type, Object fieldValue);
- void updateEntity(EntityEnum type, BaseEntityModel model);
-
- void removeEntity(EntityEnum type, Long id);
-}
+import { TestBed } from '@angular/core/testing';
+import { RouterTestingModule } from '@angular/router/testing';
+import { AppComponent } from './app.component';
+
+describe('AppComponent', () => {
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ imports: [
+ RouterTestingModule
+ ],
+ declarations: [
+ AppComponent
+ ],
+ }).compileComponents();
+ });
+
+ it('should create the app', () => {
+ const fixture = TestBed.createComponent(AppComponent);
+ const app = fixture.componentInstance;
+ expect(app).toBeTruthy();
+ });
+
+ it(`should contain 'monitoring' in title`, () => {
+ const fixture = TestBed.createComponent(AppComponent);
+ const app = fixture.componentInstance;
+ expect(app.title).toContain('monitoring');
+ });
+});
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/app.component.ts
similarity index 79%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/app.component.ts
index 18b17ea7fc..148b87e90c 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/app.component.ts
@@ -17,10 +17,13 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+import { Component } from '@angular/core';
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+@Component({
+ selector: 'app-root',
+ templateUrl: './app.component.html',
+ styleUrls: ['./app.component.scss']
+})
+export class AppComponent {
+ title = 'monitoring-ui';
}
diff --git a/scripts/monitoring/src/app/app.module.ts b/scripts/monitoring/src/app/app.module.ts
new file mode 100644
index 0000000000..261ae6ebba
--- /dev/null
+++ b/scripts/monitoring/src/app/app.module.ts
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { NgModule, } from '@angular/core';
+import { BrowserModule } from '@angular/platform-browser';
+import { FormsModule, ReactiveFormsModule } from '@angular/forms';
+
+import { AppRoutingModule } from './app-routing.module';
+import { AppComponent } from './app.component';
+import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
+import { LayoutComponent } from './modules/layout/layout.component';
+import { DashboardComponent } from './modules/dashboard/main/dashboard.component';
+
+import { HttpClientModule } from '@angular/common/http';
+
+import { MaterialModule } from './material.module'
+
+import { DragDropModule } from '@angular/cdk/drag-drop';
+import { CoordinatorComponent } from './modules/dashboard/coordinator/coordinator.component';
+import { ConnectionComponent } from './modules/dashboard/connection/connection.component';
+import { WorkerComponent } from './modules/dashboard/worker/worker.component';
+import { DialogDashboardComponent } from './modules/dashboard/dialog-dashboard/dialog-dashboard.component';
+import { DashboardDirective } from './modules/dashboard/main/dashboard.directive';
+import { ListCoordinatorsComponent } from './modules/coordinators/list/list.component';
+import { ListWorkersComponent } from './modules/workers/list/list.component';
+import { ViewWorkerComponent } from './modules/workers/view/view.component';
+import { CreateEditCoordinatorsComponent } from "./modules/coordinators/create-edit/create-edit.component";
+import { CreateEditWorkersComponent } from "./modules/workers/create-edit/create-edit.component";
+import { ListWorkersEventsComponent } from "./modules/events/list/list.component";
+import { ViewWorkerEventsComponent } from "./modules/events/view/view.component";
+
+@NgModule({
+ declarations: [
+ AppComponent,
+ LayoutComponent,
+ DashboardComponent,
+ CoordinatorComponent,
+ ConnectionComponent,
+ WorkerComponent,
+ DialogDashboardComponent,
+ DashboardDirective,
+ ListCoordinatorsComponent,
+ ListWorkersComponent,
+ ViewWorkerComponent,
+ CreateEditCoordinatorsComponent,
+ CreateEditWorkersComponent,
+ ListWorkersEventsComponent,
+ ViewWorkerEventsComponent
+ ],
+ imports: [
+ BrowserModule,
+ HttpClientModule,
+ AppRoutingModule,
+ BrowserAnimationsModule,
+ MaterialModule,
+ DragDropModule,
+ FormsModule,
+ ReactiveFormsModule,
+ ],
+ providers: [],
+ bootstrap: [AppComponent]
+})
+export class AppModule {
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java b/scripts/monitoring/src/app/constants.ts
similarity index 56%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
copy to scripts/monitoring/src/app/constants.ts
index dd683080e2..da4714c939 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
+++ b/scripts/monitoring/src/app/constants.ts
@@ -17,22 +17,34 @@
* under the License.
*/
+const BASE_URI = 'http://localhost:8080';
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
-
-import java.util.List;
-
-public interface IRepository {
- Long createEntity(EntityEnum type, BaseEntityModel model);
-
- BaseEntityModel getEntity(EntityEnum type, Long id);
+let uriParts = {
+ dashboard: '/dashboard',
+ coordinators: BASE_URI + '/coordinators',
+ workers: BASE_URI + '/workers',
+ statistics: BASE_URI + '/statistics'
+}
- List<BaseEntityModel> getAllEntities(EntityEnum type);
+let prefixes = {
+ coordinator: 'coordinator-',
+ worker: 'worker-'
+}
- List<BaseEntityModel> getAllEntitiesByField(EntityEnum type, Object fieldValue);
- void updateEntity(EntityEnum type, BaseEntityModel model);
- void removeEntity(EntityEnum type, Long id);
+let chartColors = {
+ red: 'rgb(255, 99, 132)',
+ orange: 'rgb(255, 159, 64)',
+ yellow: 'rgb(255, 205, 86)',
+ green: 'rgb(75, 192, 192)',
+ blue: 'rgb(54, 162, 235)',
+ purple: 'rgb(153, 102, 255)',
+ grey: 'rgb(201, 203, 207)',
+ white: 'rgb(255, 255, 255)'
+};
+
+export const constants = {
+ uriParts: uriParts,
+ prefixes: prefixes,
+ chartColors: chartColors
}
diff --git a/scripts/monitoring/src/app/material.module.ts b/scripts/monitoring/src/app/material.module.ts
new file mode 100644
index 0000000000..8ce3f93284
--- /dev/null
+++ b/scripts/monitoring/src/app/material.module.ts
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { NgModule } from '@angular/core';
+import { CommonModule } from '@angular/common';
+// Material Form Controls
+import { MatAutocompleteModule } from '@angular/material/autocomplete';
+import { MatCheckboxModule } from '@angular/material/checkbox';
+import { MatDatepickerModule } from '@angular/material/datepicker';
+import { MatFormFieldModule } from '@angular/material/form-field';
+import { MatInputModule } from '@angular/material/input';
+import { MatRadioModule } from '@angular/material/radio';
+import { MatSelectModule } from '@angular/material/select';
+import { MatSliderModule } from '@angular/material/slider';
+import { MatSlideToggleModule } from '@angular/material/slide-toggle';
+// Material Navigation
+import { MatMenuModule } from '@angular/material/menu';
+import { MatSidenavModule } from '@angular/material/sidenav';
+import { MatToolbarModule } from '@angular/material/toolbar';
+// Material Layout
+import { MatCardModule } from '@angular/material/card';
+import { MatDividerModule } from '@angular/material/divider';
+import { MatExpansionModule } from '@angular/material/expansion';
+import { MatGridListModule } from '@angular/material/grid-list';
+import { MatListModule } from '@angular/material/list';
+import { MatStepperModule } from '@angular/material/stepper';
+import { MatTabsModule } from '@angular/material/tabs';
+import { MatTreeModule } from '@angular/material/tree';
+// Material Buttons & Indicators
+import { MatButtonModule } from '@angular/material/button';
+import { MatButtonToggleModule } from '@angular/material/button-toggle';
+import { MatBadgeModule } from '@angular/material/badge';
+import { MatChipsModule } from '@angular/material/chips';
+import { MatIconModule } from '@angular/material/icon';
+import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
+import { MatProgressBarModule } from '@angular/material/progress-bar';
+import { MatRippleModule } from '@angular/material/core';
+// Material Popups & Modals
+import { MatBottomSheetModule } from '@angular/material/bottom-sheet';
+import { MatDialogModule } from '@angular/material/dialog';
+import { MatSnackBarModule } from '@angular/material/snack-bar';
+import { MatTooltipModule } from '@angular/material/tooltip';
+// Material Data tables
+import { MatPaginatorModule } from '@angular/material/paginator';
+import { MatSortModule } from '@angular/material/sort';
+import { MatTableModule } from '@angular/material/table';
+
+@NgModule({
+ declarations: [],
+ imports: [
+ CommonModule,
+ MatAutocompleteModule,
+ MatCheckboxModule,
+ MatDatepickerModule,
+ MatFormFieldModule,
+ MatInputModule,
+ MatRadioModule,
+ MatSelectModule,
+ MatSliderModule,
+ MatSlideToggleModule,
+ MatMenuModule,
+ MatSidenavModule,
+ MatToolbarModule,
+ MatCardModule,
+ MatDividerModule,
+ MatExpansionModule,
+ MatGridListModule,
+ MatListModule,
+ MatStepperModule,
+ MatTabsModule,
+ MatTreeModule,
+ MatButtonModule,
+ MatButtonToggleModule,
+ MatBadgeModule,
+ MatChipsModule,
+ MatIconModule,
+ MatProgressSpinnerModule,
+ MatProgressBarModule,
+ MatRippleModule,
+ MatBottomSheetModule,
+ MatDialogModule,
+ MatSnackBarModule,
+ MatTooltipModule,
+ MatPaginatorModule,
+ MatSortModule,
+ MatTableModule,
+ ],
+ exports: [
+ MatAutocompleteModule,
+ MatCheckboxModule,
+ MatDatepickerModule,
+ MatFormFieldModule,
+ MatInputModule,
+ MatRadioModule,
+ MatSelectModule,
+ MatSliderModule,
+ MatSlideToggleModule,
+ MatMenuModule,
+ MatSidenavModule,
+ MatToolbarModule,
+ MatCardModule,
+ MatDividerModule,
+ MatExpansionModule,
+ MatGridListModule,
+ MatListModule,
+ MatStepperModule,
+ MatTabsModule,
+ MatTreeModule,
+ MatButtonModule,
+ MatButtonToggleModule,
+ MatBadgeModule,
+ MatChipsModule,
+ MatIconModule,
+ MatProgressSpinnerModule,
+ MatProgressBarModule,
+ MatRippleModule,
+ MatBottomSheetModule,
+ MatDialogModule,
+ MatSnackBarModule,
+ MatTooltipModule,
+ MatPaginatorModule,
+ MatSortModule,
+ MatTableModule
+ ]
+})
+export class MaterialModule {
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/models/coordinator.model.ts
similarity index 83%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/models/coordinator.model.ts
index 18b17ea7fc..e316a8b5e5 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/models/coordinator.model.ts
@@ -17,10 +17,9 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+export class Coordinator {
+ constructor(public id: number = -1,
+ public name: string = '',
+ public host: string = '',
+ public processId: number = -1) {}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/models/dataObject.model.ts
similarity index 82%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/models/dataObject.model.ts
index 18b17ea7fc..708e0f30fa 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/models/dataObject.model.ts
@@ -17,10 +17,9 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+export class DataObject {
+ constructor(public varName: string = '',
+ public dataType: string = '',
+ public valueType: string = '',
+ public size: number = 0) { }
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/models/event.model.ts
similarity index 83%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/models/event.model.ts
index 18b17ea7fc..62aac57347 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/models/event.model.ts
@@ -17,10 +17,9 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+import { EventStage } from "./eventStage.model";
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+export class Event {
+ constructor(public coordinatorName: string = '',
+ public stages: EventStage[] = []) { }
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java b/scripts/monitoring/src/app/models/eventStage.model.ts
similarity index 84%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
copy to scripts/monitoring/src/app/models/eventStage.model.ts
index 41cf507696..096a63f5c2 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
+++ b/scripts/monitoring/src/app/models/eventStage.model.ts
@@ -17,6 +17,8 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-
-public abstract class BaseEntityModel { }
+export class EventStage {
+ constructor(public operation: string = '',
+ public startTime: string = '',
+ public endTime: string = '') { }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java b/scripts/monitoring/src/app/models/fedRequest.model.ts
similarity index 87%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
copy to scripts/monitoring/src/app/models/fedRequest.model.ts
index 41cf507696..59b4c1768f 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
+++ b/scripts/monitoring/src/app/models/fedRequest.model.ts
@@ -17,6 +17,7 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-
-public abstract class BaseEntityModel { }
+export class FedRequest {
+ constructor(public type: string = '',
+ public count: number = 0) { }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/models/fedSiteData.model.ts
similarity index 82%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/models/fedSiteData.model.ts
index 18b17ea7fc..ff59be5543 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/models/fedSiteData.model.ts
@@ -17,10 +17,10 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+import { Coordinator } from "./coordinator.model";
+import { Worker } from "./worker.model";
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+export interface FedSiteData {
+ coordinators: Coordinator[];
+ workers: Worker[];
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/models/statistics.model.ts
similarity index 63%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/models/statistics.model.ts
index 18b17ea7fc..ae2c19ef10 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/models/statistics.model.ts
@@ -17,10 +17,16 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+import { Utilization } from "./utilization.model";
+import { Traffic } from "./traffic.model";
+import { Event } from "./event.model";
+import { DataObject } from "./dataObject.model";
+import { FedRequest } from "./fedRequest.model";
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+export class Statistics {
+ constructor(public utilization: Utilization[] = [],
+ public traffic: Traffic[] = [],
+ public events: Event[] = [],
+ public dataObjects: DataObject[] = [],
+ public requests: FedRequest[] = []) { }
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/models/traffic.model.ts
similarity index 84%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/models/traffic.model.ts
index 18b17ea7fc..ce27041b24 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/models/traffic.model.ts
@@ -17,10 +17,8 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+export class Traffic {
+ constructor(public timestamp: string = '',
+ public coordinatorId: number = -1,
+ public byteAmount: number = 0) { }
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/models/utilization.model.ts
similarity index 84%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/models/utilization.model.ts
index 18b17ea7fc..51584f6a10 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/models/utilization.model.ts
@@ -17,10 +17,8 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+export class Utilization {
+ constructor(public timestamp: string = '',
+ public cpuUsage: number = 0,
+ public memoryUsage: number = 0) { }
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/models/worker.model.ts
similarity index 83%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/models/worker.model.ts
index 18b17ea7fc..7b59a1464e 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/models/worker.model.ts
@@ -17,10 +17,10 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+export class Worker {
+ constructor(public id: number = -1,
+ public name: string = '',
+ public address: string = '',
+ public isOnline: boolean = false) { }
}
diff --git a/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.html b/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.html
new file mode 100644
index 0000000000..ad00162d04
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.html
@@ -0,0 +1,43 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<h1 mat-dialog-title>Register new coordinator</h1>
+<div id="register-coordinator-content" mat-dialog-content>
+
+ <mat-form-field appearance="fill">
+ <mat-label>Name</mat-label>
+ <input *ngIf="model" [(ngModel)]="model.name" matInput>
+ </mat-form-field>
+
+ <mat-form-field appearance="fill">
+ <mat-label>Host</mat-label>
+ <input *ngIf="model" [(ngModel)]="model.host" matInput>
+ </mat-form-field>
+
+ <mat-form-field appearance="fill">
+ <mat-label>Process Id</mat-label>
+ <input *ngIf="model" [(ngModel)]="model.processId" matInput>
+ </mat-form-field>
+
+
+</div>
+<div mat-dialog-actions>
+ <button mat-button (click)="onCancelClick()">Cancel</button>
+ <button mat-button (click)="onSaveClick()">Save</button>
+</div>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java b/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.scss
similarity index 87%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
copy to scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.scss
index 41cf507696..6e8866c7c2 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
+++ b/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.scss
@@ -17,6 +17,6 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-
-public abstract class BaseEntityModel { }
+.mat-form-field {
+ display: block;
+}
diff --git a/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.spec.ts b/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.spec.ts
new file mode 100644
index 0000000000..c8397d98a3
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.spec.ts
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CreateEditCoordinatorsComponent } from './create-edit.component';
+import { DebugElement } from "@angular/core";
+import { By } from "@angular/platform-browser";
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { FederatedSiteServiceStub } from "../../../services/federatedSiteService.stub";
+import { MAT_DIALOG_DATA, MatDialogRef } from "@angular/material/dialog";
+
+describe('CreateEditCoordinatorsComponent', () => {
+ let component: CreateEditCoordinatorsComponent;
+ let fixture: ComponentFixture<CreateEditCoordinatorsComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [ CreateEditCoordinatorsComponent ],
+ providers: [
+ { provide: FederatedSiteService , useClass: FederatedSiteServiceStub },
+ { provide : MAT_DIALOG_DATA, useValue : {} },
+ { provide: MatDialogRef, useValue: {} }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(CreateEditCoordinatorsComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should not contain null model', () => {
+ expect(component.model).not.toBeNull();
+ });
+
+ it('should contain name and address fields', () => {
+ let html = de.query(By.css('#register-coordinator-content')).nativeElement.innerText;
+ expect(html).toContain('Name');
+ expect(html).toContain('Host');
+ expect(html).toContain('Monitoring Id');
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.ts b/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.ts
new file mode 100644
index 0000000000..11136e190e
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/coordinators/create-edit/create-edit.component.ts
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, Inject } from '@angular/core';
+import { Coordinator } from "../../../models/coordinator.model";
+import { MAT_DIALOG_DATA, MatDialogRef } from "@angular/material/dialog";
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+
+@Component({
+ selector: 'app-create-edit-coordinator',
+ templateUrl: './create-edit.component.html',
+ styleUrls: ['./create-edit.component.scss']
+})
+export class CreateEditCoordinatorsComponent {
+
+ public model: Coordinator;
+
+ constructor(
+ private fedSiteService: FederatedSiteService,
+ public dialogRef: MatDialogRef<CreateEditCoordinatorsComponent>,
+ @Inject(MAT_DIALOG_DATA) public id: number) {
+ }
+
+ ngOnInit(): void {
+ this.model = new Coordinator();
+
+ if (this.id !== null) {
+ this.fedSiteService.getCoordinator(this.id).subscribe(coordinator => this.model = coordinator);
+ }
+ }
+
+ onSaveClick() {
+
+ if (this.id !== null) {
+ this.fedSiteService.editCoordinator(this.model).subscribe(coordinator => this.model = coordinator);
+ } else {
+ this.fedSiteService.createCoordinator(this.model).subscribe(coordinator => this.model = coordinator);
+ }
+
+ this.dialogRef.close()
+ }
+
+ onCancelClick() {
+ this.dialogRef.close()
+ }
+}
diff --git a/scripts/monitoring/src/app/modules/coordinators/list/list.component.html b/scripts/monitoring/src/app/modules/coordinators/list/list.component.html
new file mode 100644
index 0000000000..128d718af0
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/coordinators/list/list.component.html
@@ -0,0 +1,71 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<div class="container" fxLayout="row" fxLayoutAlign="center none">
+ <div fxFlex="95%">
+
+ <mat-card>
+
+ <mat-card-header>
+ <h2>Coordinators</h2>
+ <button [ngClass]="[ loadingData ? 'loading' : '']" (click)="refreshData()" color="warn" mat-mini-fab>
+ <mat-icon aria-hidden="false" aria-label="refresh">refresh</mat-icon>
+ </button>
+ </mat-card-header>
+ <mat-card-content>
+
+ <table [dataSource]="dataSource" mat-table matSort>
+
+ <ng-container matColumnDef="name">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Name</th>
+ <td *matCellDef="let element" mat-cell> {{element.name}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="host">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Host</th>
+ <td *matCellDef="let element" mat-cell> {{element.host}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="processId">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Process Id</th>
+ <td *matCellDef="let element" mat-cell> {{element.processId}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="actions">
+ <th *matHeaderCellDef mat-header-cell> Actions</th>
+ <td *matCellDef="let element" mat-cell>
+ <button (click)="editCoordinator(element.id)" color="accent" mat-mini-fab>
+ <mat-icon aria-hidden="false" aria-label="edit">edit</mat-icon>
+ </button>
+ <button (click)="deleteCoordinator(element.id)" color="warn" mat-mini-fab>
+ <mat-icon aria-hidden="false" aria-label="delete">delete</mat-icon>
+ </button>
+ </td>
+ </ng-container>
+
+ <tr *matHeaderRowDef="displayedColumns" mat-header-row></tr>
+ <tr *matRowDef="let row; columns: displayedColumns;" mat-row></tr>
+
+ </table>
+
+ </mat-card-content>
+ </mat-card>
+
+ </div>
+</div>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/modules/coordinators/list/list.component.scss
similarity index 65%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/modules/coordinators/list/list.component.scss
index 18b17ea7fc..bbc9d83ade 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/modules/coordinators/list/list.component.scss
@@ -17,10 +17,43 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+table {
+ width: 100%;
+}
+
+th.mat-sort-header-sorted {
+ color: black;
+}
+
+mat-card-header {
+ display: flex;
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+ button {
+ margin-left: auto;
+ }
+}
+
+.container {
+ margin: 2em;
+}
+
+.demo-table {
+ width: 100%;
+}
+
+button {
+ margin: 0.5em 0.5em 0.5em 0;
+}
+
+.loading {
+ animation: rotation 2s infinite linear;
+ filter: brightness(1.5);
+}
+@keyframes rotation {
+ from {
+ transform: rotate(0deg);
+ }
+ to {
+ transform: rotate(359deg);
+ }
}
diff --git a/scripts/monitoring/src/app/modules/coordinators/list/list.component.spec.ts b/scripts/monitoring/src/app/modules/coordinators/list/list.component.spec.ts
new file mode 100644
index 0000000000..a55d96b404
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/coordinators/list/list.component.spec.ts
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ListCoordinatorsComponent } from './list.component';
+import { DebugElement } from "@angular/core";
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { FederatedSiteServiceStub } from "../../../services/federatedSiteService.stub";
+import { MatDialog } from "@angular/material/dialog";
+import { Router } from "@angular/router";
+import { By } from "@angular/platform-browser";
+
+describe('ListCoordinatorsComponent', () => {
+ let component: ListCoordinatorsComponent;
+ let fixture: ComponentFixture<ListCoordinatorsComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [ListCoordinatorsComponent],
+ providers: [
+ { provide: FederatedSiteService , useClass: FederatedSiteServiceStub },
+ { provide : Router, useValue : {} },
+ { provide: MatDialog, useValue: {} }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(ListCoordinatorsComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should contain table of coordinators', () => {
+ expect(de.query(By.css('table'))).not.toBeNull();
+ });
+
+ it('should contain name, address and actions table fields', () => {
+ expect(component.displayedColumns).toContain('name');
+ expect(component.displayedColumns).toContain('address');
+ expect(component.displayedColumns).toContain('actions');
+ });
+
+ it('should not have null data source', () => {
+ expect(component.dataSource).not.toBeNull();
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/coordinators/list/list.component.ts b/scripts/monitoring/src/app/modules/coordinators/list/list.component.ts
new file mode 100644
index 0000000000..59d38b1d67
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/coordinators/list/list.component.ts
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, ViewChild } from '@angular/core';
+import { MatSort } from '@angular/material/sort';
+import { Router } from '@angular/router';
+
+import { Coordinator } from 'src/app/models/coordinator.model';
+import { FederatedSiteService } from 'src/app/services/federatedSiteService.service';
+import { MatTableDataSource } from "@angular/material/table";
+import { MatDialog } from "@angular/material/dialog";
+import { CreateEditCoordinatorsComponent } from "../create-edit/create-edit.component";
+
+@Component({
+ selector: 'app-list-coordinators',
+ templateUrl: './list.component.html',
+ styleUrls: ['./list.component.scss']
+})
+export class ListCoordinatorsComponent {
+
+ public displayedColumns: string[] = ['name', 'host', 'processId', 'actions'];
+ public dataSource: MatTableDataSource<Coordinator> = new MatTableDataSource<Coordinator>([]);
+
+ @ViewChild(MatSort, {static: true})
+ sort: MatSort = new MatSort;
+
+ public loadingData: boolean = false;
+
+ constructor(
+ public dialog: MatDialog,
+ private fedSiteService: FederatedSiteService) {
+ }
+
+ ngOnInit(): void {
+ this.refreshData();
+ }
+
+ editCoordinator(id: number) {
+ this.dialog.open(CreateEditCoordinatorsComponent, {
+ width: '500px',
+ data: id
+ });
+ }
+
+ deleteCoordinator(id: number) {
+ this.fedSiteService.deleteCoordinator(id).subscribe(() => {
+ this.dataSource.data = this.dataSource.data.filter(c => c.id !== id)
+ });
+ }
+
+ refreshData() {
+ this.loadingData = true;
+ this.fedSiteService.getAllCoordinators().subscribe(coordinators => {
+ this.dataSource.data = coordinators
+ this.loadingData = false;
+ });
+ }
+
+}
diff --git a/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.html b/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.html
new file mode 100644
index 0000000000..7effedb715
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.html
@@ -0,0 +1,22 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<mat-card class="connection-card">
+ <canvas></canvas>
+</mat-card>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java b/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.scss
similarity index 87%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
copy to scripts/monitoring/src/app/modules/dashboard/connection/connection.component.scss
index 41cf507696..dff4046918 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
+++ b/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.scss
@@ -17,6 +17,7 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-
-public abstract class BaseEntityModel { }
+.connection-card {
+ width: 220px;
+ height: 120px;
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java b/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.spec.ts
similarity index 56%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
copy to scripts/monitoring/src/app/modules/dashboard/connection/connection.component.spec.ts
index dd683080e2..9e7f4c46ff 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
+++ b/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.spec.ts
@@ -17,22 +17,28 @@
* under the License.
*/
-
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
-
-import java.util.List;
-
-public interface IRepository {
- Long createEntity(EntityEnum type, BaseEntityModel model);
-
- BaseEntityModel getEntity(EntityEnum type, Long id);
-
- List<BaseEntityModel> getAllEntities(EntityEnum type);
-
- List<BaseEntityModel> getAllEntitiesByField(EntityEnum type, Object fieldValue);
- void updateEntity(EntityEnum type, BaseEntityModel model);
-
- void removeEntity(EntityEnum type, Long id);
-}
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ConnectionComponent } from './connection.component';
+
+describe('ConnectionComponent', () => {
+ let component: ConnectionComponent;
+ let fixture: ComponentFixture<ConnectionComponent>;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [ConnectionComponent]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(ConnectionComponent);
+ component = fixture.componentInstance;
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.ts b/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.ts
new file mode 100644
index 0000000000..883eb89a64
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/connection/connection.component.ts
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, OnInit } from '@angular/core';
+import { Chart } from "chart.js";
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { Worker } from "../../../models/worker.model";
+import { Coordinator } from "../../../models/coordinator.model";
+import { Statistics } from "../../../models/statistics.model";
+import { constants } from "../../../constants";
+import 'chartjs-adapter-moment';
+import { Subject } from "rxjs";
+import { Utils } from "../../../utils";
+
+@Component({
+ selector: 'app-connection',
+ templateUrl: './connection.component.html',
+ styleUrls: ['./connection.component.scss']
+})
+export class ConnectionComponent implements OnInit {
+
+ public workerId: number;
+
+ public worker: Worker;
+ public coordinator: Coordinator;
+ public statistics: Statistics;
+
+ private stopPollingStatistics = new Subject<any>();
+
+ constructor(private fedSiteService: FederatedSiteService) { }
+
+ ngOnInit(): void {
+ this.statistics = new Statistics();
+
+ const id = `traffic-${constants.prefixes.coordinator + this.coordinator.id + constants.prefixes.worker + this.worker.id}`;
+
+ const trafficMetricEle: any = document.getElementById(id);
+
+ let trafficChart = new Chart(trafficMetricEle.getContext('2d'), {
+ type: 'line',
+ data: {
+ datasets: [{
+ data: this.statistics.utilization.map(s => {
+ return { x: s.timestamp, y: s.memoryUsage }
+ }),
+ borderColor: constants.chartColors.green
+ }]
+ },
+ options: {
+ responsive: true,
+ plugins: {
+ legend: {
+ display: false
+ },
+ title: {
+ display: true,
+ text: 'I/O Bytes'
+ }
+ },
+ scales: {
+ x: {
+ grid: {
+ display: false
+ },
+ type: 'timeseries',
+ ticks: {
+ display: false,
+ }
+ },
+ y: {
+ beginAtZero: true,
+ }
+ }
+ },
+ });
+
+ this.fedSiteService.getStatisticsPolling(this.workerId, this.stopPollingStatistics).subscribe(stats => {
+ this.statistics = stats;
+
+ trafficChart.data.datasets.forEach((dataset) => {
+ dataset.data = [];
+ this.statistics.traffic.map(s => dataset.data.push({ x: s.timestamp, y: s.byteAmount }));
+ dataset.data.sort(Utils.sortTimestamp);
+ });
+
+ trafficChart.update();
+ });
+ }
+
+ ngOnDestroy() {
+ this.stopPollingStatistics.next(null);
+ }
+}
diff --git a/scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.html b/scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.html
new file mode 100644
index 0000000000..49ea1c1201
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.html
@@ -0,0 +1,40 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<mat-card class="coordinator-card">
+ <mat-card-title *ngIf="model">{{ model.name }}</mat-card-title>
+ <mat-divider inset></mat-divider>
+ <mat-card-content *ngIf="model">
+ <div class="info-row">
+ <mat-icon mat-list-icon>
+ laptop
+ </mat-icon>
+ <p> <span class="bold">Host:</span> {{ model.host }} </p>
+ </div>
+ </mat-card-content>
+ <mat-divider inset></mat-divider>
+ <mat-card-content *ngIf="model">
+ <div class="info-row">
+ <mat-icon mat-list-icon>
+ memory
+ </mat-icon>
+ <p> <span class="bold">Process id:</span> {{ model.processId }} </p>
+ </div>
+ </mat-card-content>
+</mat-card>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.scss
similarity index 82%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.scss
index 18b17ea7fc..46ac60c431 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.scss
@@ -17,10 +17,20 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+.coordinator-card {
+ width: 200px;
+ height: 100px;
+}
+
+.bold {
+ font-weight: bold;
+}
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+.info-row {
+ display: inline;
+ margin: auto;
+ padding: 0.2em;
+ > * {
+ float: left;
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java b/scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.spec.ts
similarity index 52%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
copy to scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.spec.ts
index dd683080e2..a1fa7acf30 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
+++ b/scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.spec.ts
@@ -17,22 +17,32 @@
* under the License.
*/
-
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
-
-import java.util.List;
-
-public interface IRepository {
- Long createEntity(EntityEnum type, BaseEntityModel model);
-
- BaseEntityModel getEntity(EntityEnum type, Long id);
-
- List<BaseEntityModel> getAllEntities(EntityEnum type);
-
- List<BaseEntityModel> getAllEntitiesByField(EntityEnum type, Object fieldValue);
- void updateEntity(EntityEnum type, BaseEntityModel model);
-
- void removeEntity(EntityEnum type, Long id);
-}
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CoordinatorComponent } from './coordinator.component';
+import { DebugElement } from "@angular/core";
+
+describe('CoordinatorComponent', () => {
+ let component: CoordinatorComponent;
+ let fixture: ComponentFixture<CoordinatorComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [CoordinatorComponent]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(CoordinatorComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+});
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.ts
similarity index 69%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.ts
index 18b17ea7fc..036ad818b5 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/modules/dashboard/coordinator/coordinator.component.ts
@@ -17,10 +17,21 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+import { Component } from '@angular/core';
+
+import { Coordinator } from 'src/app/models/coordinator.model';
+
+@Component({
+ selector: 'app-coordinator',
+ templateUrl: './coordinator.component.html',
+ styleUrls: ['./coordinator.component.scss']
+})
+export class CoordinatorComponent {
+
+ public model: Coordinator;
+
+ constructor() { }
+
+ ngOnInit(): void { }
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
}
diff --git a/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.html b/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.html
new file mode 100644
index 0000000000..dd49ac9cdd
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.html
@@ -0,0 +1,46 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<h1 mat-dialog-title>Choose which coordinators and workers to show</h1>
+
+<div id="dashboard-dialog-content" mat-dialog-content>
+ <h4>Coordinators:</h4>
+ <section *ngFor="let coordinator of data.coordinators" class="dialog-dashboard-section">
+ <span class="dialog-dashboard-list-section">
+ <mat-checkbox (change)="changeSelectedCoordinators(coordinator.id)" class="dialog-dashboard-margin">
+ {{coordinator.name}}
+ </mat-checkbox>
+ </span>
+ </section>
+ <hr/>
+
+ <h4>Workers:</h4>
+ <section *ngFor="let worker of data.workers" class="dialog-dashboard-section">
+ <span class="dialog-dashboard-list-section">
+ <mat-checkbox (change)="changeSelectedWorkers(worker.id)" class="dialog-dashboard-margin">
+ {{worker.name}}
+ </mat-checkbox>
+ </span>
+ </section>
+</div>
+
+<div mat-dialog-actions>
+ <button (click)="onCancelClick()" mat-button>Cancel</button>
+ <button (click)="onSaveClick()" cdkFocusInitial mat-button>Save</button>
+</div>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.scss
similarity index 84%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.scss
index 18b17ea7fc..92562a34ca 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.scss
@@ -17,10 +17,15 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+.dialog-dashboard-section {
+ margin: 12px 0;
+}
+
+.dialog-dashboard-margin {
+ margin: 0 12px;
+}
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+ul {
+ list-style-type: none;
+ margin-top: 4px;
}
diff --git a/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.spec.ts b/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.spec.ts
new file mode 100644
index 0000000000..4b63d06874
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.spec.ts
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { DialogDashboardComponent } from './dialog-dashboard.component';
+import { MAT_DIALOG_DATA, MatDialogRef } from "@angular/material/dialog";
+import { By } from "@angular/platform-browser";
+import { DebugElement } from "@angular/core";
+
+describe('DialogDashboardComponent', () => {
+ let component: DialogDashboardComponent;
+ let fixture: ComponentFixture<DialogDashboardComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [DialogDashboardComponent],
+ providers: [
+ { provide : MAT_DIALOG_DATA, useValue : {} },
+ { provide: MatDialogRef, useValue: {} }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(DialogDashboardComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should contain coordinators and workers selection', () => {
+ let html = de.query(By.css('#dashboard-dialog-content')).nativeElement.innerText;
+ expect(html).toContain('Coordinators');
+ expect(html).toContain('Workers');
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.ts b/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.ts
new file mode 100644
index 0000000000..23af8b4ec2
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.ts
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, Inject } from '@angular/core';
+import { DashboardComponent } from '../main/dashboard.component';
+import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { FedSiteData } from 'src/app/models/fedSiteData.model';
+
+@Component({
+ selector: 'app-dialog-dashboard',
+ templateUrl: './dialog-dashboard.component.html',
+ styleUrls: ['./dialog-dashboard.component.scss']
+})
+export class DialogDashboardComponent {
+
+ private selectedCoordinatorIds: number[] = [];
+ private selectedWorkerIds: number[] = [];
+
+ constructor(
+ public dialogRef: MatDialogRef<DashboardComponent>,
+ @Inject(MAT_DIALOG_DATA) public data: FedSiteData
+ ) {
+ }
+
+ changeSelectedCoordinators(id: number): void {
+ if (this.selectedCoordinatorIds.some(c => c === id)) {
+ this.selectedCoordinatorIds = this.selectedCoordinatorIds.filter(c => c !== id);
+ } else {
+ this.selectedCoordinatorIds.push(id);
+ }
+ }
+
+ changeSelectedWorkers(id: number): void {
+ if (this.selectedWorkerIds.some(w => w === id)) {
+ this.selectedWorkerIds = this.selectedWorkerIds.filter(w => w !== id);
+ } else {
+ this.selectedWorkerIds.push(id);
+ }
+ }
+
+ onSaveClick(): void {
+
+ this.dialogRef.close({
+ selectedWorkerIds: this.selectedWorkerIds,
+ selectedCoordinatorIds: this.selectedCoordinatorIds
+ });
+ }
+
+ onCancelClick(): void {
+ this.dialogRef.close();
+ }
+}
diff --git a/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.html b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.html
new file mode 100644
index 0000000000..69587234ff
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<button (click)="openConfigDialog()" class="md-btn-right" color="primary" mat-raised-button>Config</button>
+<div id="dashboard-content">
+
+ <ng-template fedSiteHost></ng-template>
+</div>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.scss
similarity index 77%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.scss
index 18b17ea7fc..7f043c1a0a 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.scss
@@ -17,10 +17,20 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+#dashboard-content {
+ display: inline-block;
+ width: 82%;
+ min-height: 100%;
+ box-sizing: border-box;
+ float: left;
+ position:relative;
+ margin: 0;
+ padding: 3px;
+}
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+.md-btn-right {
+ position: absolute;
+ right: 1em;
+ top: 1em;
+ margin: 0;
}
diff --git a/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.spec.ts b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.spec.ts
new file mode 100644
index 0000000000..429d289320
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.spec.ts
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { DashboardComponent } from './dashboard.component';
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { FederatedSiteServiceStub } from "../../../services/federatedSiteService.stub";
+import { MatDialog } from "@angular/material/dialog";
+import { DebugElement } from "@angular/core";
+import { By } from "@angular/platform-browser";
+
+describe('DashboardComponent', () => {
+ let component: DashboardComponent;
+ let fixture: ComponentFixture<DashboardComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [DashboardComponent],
+ providers: [
+ { provide: FederatedSiteService , useClass: FederatedSiteServiceStub },
+ { provide: MatDialog, useValue: {} }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(DashboardComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should contain config button', () => {
+ expect(de.query(By.css('button')).nativeElement.innerText.toLowerCase()).toContain('config');
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.ts b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.ts
new file mode 100644
index 0000000000..b0336cdbba
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.component.ts
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, OnInit, Output, ViewChild, ViewEncapsulation } from '@angular/core';
+import { MatDialog } from '@angular/material/dialog';
+
+import { jsPlumb, jsPlumbInstance } from 'jsplumb';
+import { constants } from 'src/app/constants';
+import { FederatedSiteService } from 'src/app/services/federatedSiteService.service';
+import { CoordinatorComponent } from '../coordinator/coordinator.component';
+import { DialogDashboardComponent } from '../dialog-dashboard/dialog-dashboard.component';
+import { WorkerComponent } from '../worker/worker.component';
+import { DashboardDirective } from './dashboard.directive';
+import { FedSiteData } from "../../../models/fedSiteData.model";
+import { Coordinator } from "../../../models/coordinator.model";
+import { Worker } from "../../../models/worker.model";
+import { ConnectionComponent } from "../connection/connection.component";
+
+@Component({
+ selector: 'app-dashboard',
+ templateUrl: './dashboard.component.html',
+ styleUrls: ['./dashboard.component.scss']
+})
+export class DashboardComponent implements OnInit {
+
+ public fedSiteData: FedSiteData;
+
+ @ViewChild(DashboardDirective, {static: true}) fedSiteHost!: DashboardDirective;
+
+ private jsPlumbInstance: jsPlumbInstance;
+
+ constructor(public dialog: MatDialog,
+ private fedSiteService: FederatedSiteService) {
+ }
+
+ ngOnInit(): void {
+
+ this.fedSiteData = {
+ workers: [],
+ coordinators: []
+ };
+
+ this.jsPlumbInstance = jsPlumb.getInstance();
+ this.jsPlumbInstance.setContainer('dashboard-content');
+
+ this.openConfigDialog();
+ }
+
+ openConfigDialog(): void {
+
+ this.fedSiteService.getAllCoordinators().subscribe(coordinators => this.fedSiteData.coordinators = coordinators);
+ this.fedSiteService.getAllWorkers().subscribe(workers => this.fedSiteData.workers = workers);
+
+ const dialogRef = this.dialog.open(DialogDashboardComponent, {
+ width: '500px',
+ data: this.fedSiteData,
+ });
+
+ dialogRef.afterClosed().subscribe(result => {
+ if (result) {
+ let selectedCoordinators = this.fedSiteData.coordinators.filter(c => result['selectedCoordinatorIds'].includes(c.id));
+ let selectedWorkers = this.fedSiteData.workers.filter(w => result['selectedWorkerIds'].includes(w.id));
+
+ this.fedSiteHost.viewContainerRef.clear();
+ this.jsPlumbInstance.removeAllEndpoints('dashboard-content');
+
+ this.redrawDiagram(selectedCoordinators, selectedWorkers);
+ }
+ });
+ }
+
+ private redrawDiagram(selectedCoordinators: Coordinator[], selectedWorkers: Worker[]) {
+
+ for (const worker of selectedWorkers) {
+ const workerComponentRef = this.fedSiteHost.viewContainerRef.createComponent(WorkerComponent);
+ workerComponentRef.instance.workerId = worker.id;
+ workerComponentRef.location.nativeElement.id = constants.prefixes.worker + worker.id;
+ workerComponentRef.location.nativeElement.style = 'position: absolute;';
+
+ this.jsPlumbInstance.draggable(constants.prefixes.worker + worker.id);
+
+ this.fedSiteService.getStatistics(worker.id).subscribe(stats => {
+
+ let coordinators = selectedCoordinators.filter(c => stats.traffic.find(ts => ts.coordinatorId === c.id))
+
+ coordinators.forEach(c => {
+ const connectionComponentRef = this.fedSiteHost.viewContainerRef.createComponent(ConnectionComponent);
+ const id = constants.prefixes.coordinator + c.id + constants.prefixes.worker + worker.id;
+ connectionComponentRef.instance.workerId = worker.id;
+ connectionComponentRef.instance.worker = worker;
+ connectionComponentRef.instance.coordinator = c;
+ connectionComponentRef.location.nativeElement.id = id;
+ connectionComponentRef.location.nativeElement.firstChild.firstChild.id = `traffic-${id}`;
+ connectionComponentRef.location.nativeElement.style = 'position: absolute;';
+
+ this.jsPlumbInstance.draggable(id);
+
+ this.jsPlumbInstance.connect({
+ source: constants.prefixes.coordinator + c.id,
+ target: id,
+ anchor: ['AutoDefault'],
+ endpoint: "Blank"
+ });
+
+ this.jsPlumbInstance.connect({
+ source: id,
+ target: constants.prefixes.worker + worker.id,
+ anchor: ['AutoDefault'],
+ endpoint: "Blank"
+ });
+ });
+ })
+ }
+
+ for (const coordinator of selectedCoordinators) {
+ const coordinatorComponentRef = this.fedSiteHost.viewContainerRef.createComponent(CoordinatorComponent);
+ coordinatorComponentRef.instance.model = coordinator;
+ coordinatorComponentRef.location.nativeElement.id = constants.prefixes.coordinator + coordinator.id;
+ coordinatorComponentRef.location.nativeElement.style = 'position: absolute;';
+
+ this.jsPlumbInstance.draggable(constants.prefixes.coordinator + coordinator.id);
+ }
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.directive.ts
similarity index 80%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/modules/dashboard/main/dashboard.directive.ts
index 18b17ea7fc..bbf882ec9f 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/modules/dashboard/main/dashboard.directive.ts
@@ -17,10 +17,12 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+import { Directive, ViewContainerRef } from '@angular/core';
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+@Directive({
+ selector: '[fedSiteHost]',
+})
+export class DashboardDirective {
+ constructor(public viewContainerRef: ViewContainerRef) {
+ }
}
diff --git a/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.html b/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.html
new file mode 100644
index 0000000000..9af92085b4
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.html
@@ -0,0 +1,78 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<mat-card class="worker-card">
+ <mat-card-title *ngIf="model">{{ model.name }}</mat-card-title>
+ <mat-divider inset></mat-divider>
+ <mat-card-content *ngIf="model">
+ <div class="info-row">
+ <mat-icon mat-list-icon>
+ laptop
+ </mat-icon>
+ <p> <span class="bold">Address:</span> {{ model.address }} </p>
+ </div>
+ </mat-card-content>
+ <mat-divider inset></mat-divider>
+ <mat-card-content *ngIf="model">
+ <div class="info-row">
+ <mat-icon mat-list-icon>
+ offline_pin
+ </mat-icon>
+ <p> <span class="bold">Status:</span>
+ <span *ngIf="model && model.isOnline" class="online-worker">
+ Online
+ </span>
+ <span *ngIf="model && !model.isOnline" class="offline-worker">
+ Offline
+ </span>
+ </p>
+ </div>
+ </mat-card-content>
+ <mat-divider inset></mat-divider>
+ <mat-card-content>
+ <canvas></canvas>
+ </mat-card-content>
+
+ <mat-divider inset></mat-divider>
+
+ <mat-card-content>
+ <table [dataSource]="dataSource" mat-table matSort>
+
+ <ng-container matColumnDef="instruction">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Top 3 Inst.</th>
+ <td *matCellDef="let element" mat-cell> {{element['instruction']}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="time">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Time(ms)</th>
+ <td *matCellDef="let element" mat-cell> {{element['time']}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="frequency">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Freq.</th>
+ <td *matCellDef="let element" mat-cell> {{element['frequency']}} </td>
+ </ng-container>
+
+ <tr *matHeaderRowDef="displayedColumns" mat-header-row></tr>
+ <tr *matRowDef="let row; columns: displayedColumns;" mat-row></tr>
+
+ </table>
+
+ </mat-card-content>
+</mat-card>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.scss
similarity index 73%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/modules/dashboard/worker/worker.component.scss
index 18b17ea7fc..12eb692580 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.scss
@@ -17,10 +17,35 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+.worker-card {
+ width: 300px;
+ height: 465px;
+}
+
+.worker-chart {
+ height: 100px;
+}
+
+table {
+ width: 100%;
+}
+
+.online-worker {
+ color: green;
+}
+
+.offline-worker {
+ color: darkred;
+}
+.bold {
+ font-weight: bold;
+}
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+.info-row {
+ display: inline;
+ margin: auto;
+ padding: 0.2em;
+ > * {
+ float: left;
+ }
}
diff --git a/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.spec.ts b/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.spec.ts
new file mode 100644
index 0000000000..1cfffd36dc
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.spec.ts
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { WorkerComponent } from './worker.component';
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { FederatedSiteServiceStub } from "../../../services/federatedSiteService.stub";
+import { DebugElement } from "@angular/core";
+import { By } from "@angular/platform-browser";
+
+describe('WorkerComponent', () => {
+ let component: WorkerComponent;
+ let fixture: ComponentFixture<WorkerComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [WorkerComponent],
+ providers: [ { provide: FederatedSiteService , useClass: FederatedSiteServiceStub } ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(WorkerComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should not contain null model', () => {
+ expect(component.model).not.toBeNull();
+ });
+
+ it('should contain a table for instructions', () => {
+ expect(de.query(By.css('table'))).not.toBeNull();
+ });
+
+ it('should contain address and status information', () => {
+ let html = de.query(By.css('mat-card-content')).nativeElement.innerText;
+ expect(html).toContain('Address');
+ expect(html).toContain('Status');
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.ts b/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.ts
new file mode 100644
index 0000000000..1d749c32fc
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/dashboard/worker/worker.component.ts
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component } from '@angular/core';
+import { Worker } from 'src/app/models/worker.model';
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { Statistics } from "../../../models/statistics.model";
+import { MatTableDataSource } from "@angular/material/table";
+import { Chart, registerables } from "chart.js";
+import { constants } from "../../../constants";
+import 'chartjs-adapter-moment';
+import { Subject } from "rxjs";
+import { Utils } from "../../../utils";
+
+@Component({
+ selector: 'app-worker',
+ templateUrl: './worker.component.html',
+ styleUrls: ['./worker.component.scss']
+})
+export class WorkerComponent {
+
+ public workerId: number;
+
+ public model: Worker;
+ public statistics: Statistics;
+
+ public displayedColumns: string[] = ['instruction', 'time', 'frequency'];
+ public dataSource: MatTableDataSource<any> = new MatTableDataSource<any>([]);
+
+ private stopPollingWorker = new Subject<any>();
+ private stopPollingStatistics = new Subject<any>();
+
+ constructor(private fedSiteService: FederatedSiteService) {
+ Chart.register(...registerables);
+ }
+
+ ngOnInit(): void {
+ this.statistics = new Statistics();
+
+ const memoryMetricEle: any = document.querySelector(`#${constants.prefixes.worker + this.workerId} canvas`);
+
+ let memoryChart = new Chart(memoryMetricEle.getContext('2d'), {
+ type: 'line',
+ data: {
+ datasets: [{
+ data: this.statistics.utilization.map(s => {
+ return { x: s.timestamp, y: s.memoryUsage }
+ }),
+ borderColor: constants.chartColors.red
+ }]
+ },
+ options: {
+ responsive: true,
+ plugins: {
+ legend: {
+ display: false
+ },
+ title: {
+ display: true,
+ text: 'Memory usage %'
+ }
+ },
+ scales: {
+ x: {
+ grid: {
+ display: false
+ },
+ type: 'timeseries',
+ ticks: {
+ display: false
+ }
+ },
+ y: {
+ beginAtZero: true
+ }
+ }
+ },
+ });
+
+ this.fedSiteService.getWorkerPolling(this.workerId, this.stopPollingWorker).subscribe(worker => this.model = worker);
+
+ this.fedSiteService.getStatisticsPolling(this.workerId, this.stopPollingStatistics).subscribe(stats => {
+ this.statistics = stats;
+
+ memoryChart.data.datasets.forEach((dataset) => {
+ dataset.data = [];
+ this.statistics.utilization.map(s => dataset.data.push({ x: s.timestamp, y: s.memoryUsage }));
+ dataset.data.sort(Utils.sortTimestamp);
+ });
+
+ memoryChart.update();
+
+ this.dataSource = this.parseInstructions();
+ });
+ }
+
+ private parseInstructions(): any {
+ let tmp = {};
+ let result: any[] = [];
+ this.statistics.events.forEach(e => {
+ e.stages.forEach(s => {
+ if (!tmp[s.operation]) {
+ tmp[s.operation] = {
+ frequency: 0,
+ time: 0
+ }
+ }
+
+ tmp[s.operation]['frequency'] += 1;
+ tmp[s.operation]['time'] += (new Date(s.endTime).getTime() - new Date(s.startTime).getTime());
+ })
+ });
+
+ for (const [key, value] of Object.entries(tmp)) {
+ result.push({
+ instruction: key,
+ // @ts-ignore
+ time: value['time'],
+ // @ts-ignore
+ frequency: value['frequency']
+ })
+ }
+
+ return result.sort((a,b) => b['time']-a['time']).slice(0,3);
+ }
+
+ ngOnDestroy() {
+ this.stopPollingWorker.next(null);
+ this.stopPollingStatistics.next(null);
+ }
+}
diff --git a/scripts/monitoring/src/app/modules/events/list/list.component.html b/scripts/monitoring/src/app/modules/events/list/list.component.html
new file mode 100644
index 0000000000..79be8f1fa5
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/events/list/list.component.html
@@ -0,0 +1,75 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<div class="container" fxLayout="row" fxLayoutAlign="center none">
+ <div fxFlex="95%">
+
+ <mat-card>
+
+ <mat-card-header>
+ <h2>Choose for which worker to show events</h2>
+ <button [ngClass]="[ loadingData ? 'loading' : '']" (click)="refreshData()" color="warn" mat-mini-fab>
+ <mat-icon aria-hidden="false" aria-label="refresh">refresh</mat-icon>
+ </button>
+ </mat-card-header>
+ <mat-card-content>
+
+ <table [dataSource]="dataSource" mat-table matSort>
+
+ <ng-container matColumnDef="name">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Name</th>
+ <td *matCellDef="let element" mat-cell> {{element.name}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="address">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Address</th>
+ <td *matCellDef="let element" mat-cell> {{element.address}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="status">
+ <th *matHeaderCellDef mat-header-cell> Status</th>
+ <td *matCellDef="let element" mat-cell>
+ <span *ngIf="element.isOnline" class="online-worker">
+ Online
+ </span>
+ <span *ngIf="!element.isOnline" class="offline-worker">
+ Offline
+ </span>
+ </td>
+ </ng-container>
+
+ <ng-container matColumnDef="actions">
+ <th *matHeaderCellDef mat-header-cell> Actions</th>
+ <td *matCellDef="let element" mat-cell>
+ <button (click)="viewEvent(element.id)" color="primary" mat-raised-button>
+ View event timeline
+ </button>
+ </td>
+ </ng-container>
+
+ <tr *matHeaderRowDef="displayedColumns" mat-header-row></tr>
+ <tr *matRowDef="let row; columns: displayedColumns;" mat-row></tr>
+
+ </table>
+
+ </mat-card-content>
+ </mat-card>
+
+ </div>
+</div>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java b/scripts/monitoring/src/app/modules/events/list/list.component.scss
similarity index 63%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
copy to scripts/monitoring/src/app/modules/events/list/list.component.scss
index 21d6812644..7a82040134 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
+++ b/scripts/monitoring/src/app/modules/events/list/list.component.scss
@@ -17,27 +17,51 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
+table {
+ width: 100%;
+}
-import io.netty.handler.codec.http.HttpRequest;
+th.mat-sort-header-sorted {
+ color: black;
+}
-public class Request {
- private HttpRequest _context;
- private String _body;
+mat-card-header {
+ display: flex;
- public HttpRequest getContext() {
- return _context;
+ button {
+ margin-left: auto;
}
+}
- public void setContext(final HttpRequest requestContext) {
- this._context = requestContext;
- }
+.container {
+ margin: 2em;
+}
- public String getBody() {
- return _body;
- }
+.demo-table {
+ width: 100%;
+}
- public void setBody(final String content) {
- this._body = content;
+button {
+ margin: 0.5em 0.5em 0.5em 0;
+}
+
+.online-worker {
+ color: green;
+}
+
+.offline-worker {
+ color: darkred;
+}
+
+.loading {
+ animation: rotation 2s infinite linear;
+ filter: brightness(1.5);
+}
+@keyframes rotation {
+ from {
+ transform: rotate(0deg);
+ }
+ to {
+ transform: rotate(359deg);
}
}
diff --git a/scripts/monitoring/src/app/modules/events/list/list.component.spec.ts b/scripts/monitoring/src/app/modules/events/list/list.component.spec.ts
new file mode 100644
index 0000000000..342b5fac0b
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/events/list/list.component.spec.ts
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ListWorkersComponent } from './list.component';
+import { DebugElement } from "@angular/core";
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { FederatedSiteServiceStub } from "../../../services/federatedSiteService.stub";
+import { Router } from "@angular/router";
+import { MatDialog } from "@angular/material/dialog";
+import { By } from "@angular/platform-browser";
+
+describe('ListWorkersComponent', () => {
+ let component: ListWorkersComponent;
+ let fixture: ComponentFixture<ListWorkersComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [ListWorkersComponent],
+ providers: [
+ { provide: FederatedSiteService , useClass: FederatedSiteServiceStub },
+ { provide : Router, useValue : {} },
+ { provide: MatDialog, useValue: {} }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(ListWorkersComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should contain table of workers', () => {
+ expect(de.query(By.css('table'))).not.toBeNull();
+ });
+
+ it('should contain name, address and actions table fields', () => {
+ expect(component.displayedColumns).toContain('name');
+ expect(component.displayedColumns).toContain('address');
+ expect(component.displayedColumns).toContain('actions');
+ });
+
+ it('should not have null data source', () => {
+ expect(component.dataSource).not.toBeNull();
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/events/list/list.component.ts b/scripts/monitoring/src/app/modules/events/list/list.component.ts
new file mode 100644
index 0000000000..49e74b4bdd
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/events/list/list.component.ts
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, ViewChild } from '@angular/core';
+import { MatSort } from '@angular/material/sort';
+import { Router } from '@angular/router';
+
+import { Worker } from 'src/app/models/worker.model';
+import { FederatedSiteService } from 'src/app/services/federatedSiteService.service';
+import { MatTableDataSource } from "@angular/material/table";
+import { MatDialog } from "@angular/material/dialog";
+
+@Component({
+ selector: 'app-list-workers-events',
+ templateUrl: './list.component.html',
+ styleUrls: ['./list.component.scss']
+})
+export class ListWorkersEventsComponent {
+
+ public displayedColumns: string[] = ['name', 'address', 'status', 'actions'];
+ public dataSource: MatTableDataSource<Worker> = new MatTableDataSource<Worker>([]);
+
+ public loadingData: boolean = false;
+
+ @ViewChild(MatSort, {static: true})
+ sort: MatSort = new MatSort;
+
+ constructor(
+ public dialog: MatDialog,
+ private fedSiteService: FederatedSiteService,
+ private router: Router) {
+ }
+
+ ngOnInit(): void {
+ this.refreshData();
+ }
+
+ viewEvent(workerId: number) {
+ this.router.navigate(['/events/' + workerId])
+ }
+
+ refreshData() {
+ this.loadingData = true;
+ this.fedSiteService.getAllWorkers().subscribe(workers => {
+ this.dataSource.data = workers
+ this.loadingData = false;
+ });
+ }
+
+}
diff --git a/scripts/monitoring/src/app/modules/events/view/view.component.html b/scripts/monitoring/src/app/modules/events/view/view.component.html
new file mode 100644
index 0000000000..f29c316336
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/events/view/view.component.html
@@ -0,0 +1,25 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<div class="metrics-cards">
+ <mat-card class="worker-metrics-card" id="events-metric-card">
+ <canvas id="event-timeline"></canvas>
+ </mat-card>
+</div>
+
diff --git a/scripts/monitoring/src/app/modules/events/view/view.component.scss b/scripts/monitoring/src/app/modules/events/view/view.component.scss
new file mode 100644
index 0000000000..9abbd31ea6
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/events/view/view.component.scss
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* Structure */
+.worker-container {
+ position: relative;
+ margin: 2em;
+}
+
+.worker-metrics-card {
+ margin: 2em;
+}
+
+.metrics-cards {
+ display: flex;
+
+ mat-card {
+ width: 100%;
+ }
+}
+
+.worker-table-container {
+ position: relative;
+ min-height: 200px;
+ max-height: 400px;
+ overflow: auto;
+}
+
+table {
+ width: 100%;
+}
+
+.worker-loading-shade {
+ position: absolute;
+ top: 0;
+ left: 0;
+ bottom: 56px;
+ right: 0;
+ background: rgba(0, 0, 0, 0.15);
+ z-index: 1;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+}
+
+.worker-rate-limit-reached {
+ max-width: 360px;
+ text-align: center;
+}
+
+/* Column Widths */
+.mat-column-number,
+.mat-column-state {
+ max-width: 64px;
+}
+
+.mat-column-created {
+ max-width: 124px;
+}
+
+.requests-table-title {
+ padding: 1em;
+}
+
+.online-worker {
+ color: green;
+}
+
+.offline-worker {
+ color: darkred;
+}
+
+#events-metric-card {
+ max-width: 118em;
+}
+
+#memory-metric-card {
+ margin-top: 0.5em;
+ margin-left: 0.5em;
+}
+
+#data-metric-card {
+ margin-bottom: 0.5em;
+ margin-right: 0.5em;
+}
+
+#requests-metric-card {
+ margin-bottom: 0.5em;
+ margin-left: 0.5em;
+}
diff --git a/scripts/monitoring/src/app/modules/events/view/view.component.spec.ts b/scripts/monitoring/src/app/modules/events/view/view.component.spec.ts
new file mode 100644
index 0000000000..2980cb5bc0
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/events/view/view.component.spec.ts
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ViewWorkerEventsComponent } from './view.component';
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { FederatedSiteServiceStub } from "../../../services/federatedSiteService.stub";
+import { ActivatedRoute } from "@angular/router";
+import { DebugElement } from "@angular/core";
+import { By } from "@angular/platform-browser";
+
+describe('ViewWorkerEventsComponent', () => {
+ let component: ViewWorkerEventsComponent;
+ let fixture: ComponentFixture<ViewWorkerEventsComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [ViewWorkerEventsComponent],
+ providers: [
+ { provide: FederatedSiteService , useClass: FederatedSiteServiceStub },
+ {
+ provide : ActivatedRoute,
+ useValue : {
+ snapshot: {
+ paramMap: {
+ get: () => {
+ return { id: 1 }
+ }
+ }
+ }
+ }
+ }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(ViewWorkerEventsComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should contain address, status and JIT information', () => {
+ let html = de.query(By.css('#main-worker-information')).nativeElement.innerText;
+ expect(html).toContain('Address');
+ expect(html).toContain('Status');
+ expect(html).toContain('JIT');
+ });
+
+ it('should contain CPU metrics diagram', () => {
+ expect(de.query(By.css('#cpu-metric-card'))).not.toBeNull();
+ });
+
+ it('should contain memory metrics diagram', () => {
+ expect(de.query(By.css('#memory-metric-card'))).not.toBeNull();
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/events/view/view.component.ts b/scripts/monitoring/src/app/modules/events/view/view.component.ts
new file mode 100644
index 0000000000..b7b0ebce83
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/events/view/view.component.ts
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, ViewChild } from '@angular/core';
+import { MatPaginator } from '@angular/material/paginator';
+import { MatSort } from '@angular/material/sort';
+import { ActivatedRoute } from '@angular/router';
+import { FederatedSiteService } from 'src/app/services/federatedSiteService.service';
+import { Statistics } from "../../../models/statistics.model";
+import { Chart, LegendItem, registerables } from "chart.js";
+import { constants } from "../../../constants";
+import 'chartjs-adapter-moment';
+import { Subject } from "rxjs";
+import { EventStage } from "../../../models/eventStage.model";
+
+@Component({
+ selector: 'app-view-worker-events',
+ templateUrl: './view.component.html',
+ styleUrls: ['./view.component.scss']
+})
+export class ViewWorkerEventsComponent {
+
+ public statistics: Statistics;
+
+ @ViewChild(MatPaginator) paginator: MatPaginator;
+ @ViewChild(MatSort) sort: MatSort;
+
+ private eventTimelineChart: Chart;
+
+ private stopPollingStatistics = new Subject<any>();
+
+ constructor(
+ private fedSiteService: FederatedSiteService,
+ private router: ActivatedRoute) {
+ Chart.register(...registerables);
+ }
+
+ ngOnInit(): void {
+ const id = Number(this.router.snapshot.paramMap.get('id'));
+
+ this.statistics = new Statistics();
+
+ const eventCanvasEle: any = document.getElementById('event-timeline');
+
+ this.fedSiteService.getStatisticsPolling(id, this.stopPollingStatistics).subscribe(stats => {
+ this.statistics = stats;
+
+ const timeframe = this.getTimeframe();
+ const minVal = this.getLastSeconds(timeframe[1], 3);
+
+ if (!this.eventTimelineChart) {
+ this.eventTimelineChart = new Chart(eventCanvasEle.getContext('2d'), {
+ type: 'bar',
+ data: {
+ labels: [],
+ datasets: []
+ },
+ options: {
+ indexAxis: 'y',
+ responsive: true,
+ plugins: {
+ legend: {
+ position: 'top',
+ onClick: () => null,
+ onHover: () => null,
+ onLeave: () => null,
+ labels: {
+ generateLabels(chart: Chart): LegendItem[] {
+ let legendItemsTmp: LegendItem[] = [];
+
+ for (const dataset of chart.data.datasets) {
+ const label = dataset.label!
+ if (!legendItemsTmp.find(i => i.text === label)) {
+ let li: LegendItem = {
+ text: label,
+ //@ts-ignore
+ fillStyle: dataset.backgroundColor,
+ //@ts-ignore
+ strokeStyle: dataset.borderColor,
+ }
+ legendItemsTmp.push(li);
+ }
+ }
+
+ return legendItemsTmp;
+ }
+ }
+ },
+ title: {
+ display: true,
+ text: 'Event timeline of worker with respect to coordinators'
+ }
+ },
+ scales: {
+ x: {
+ min: 0,
+ ticks: {
+ callback: function(value, index, ticks) {
+ // @ts-ignore
+ return new Date(minVal + value).toLocaleTimeString();
+ }
+ },
+ stacked: true
+ },
+ y: {
+ stacked: true
+ }
+ },
+ },
+ })
+ }
+
+ this.updateEventTimeline();
+ });
+ }
+
+ private getLastSeconds(time: number, seconds: number): number {
+ const benchmark = new Date(time);
+
+ const back = new Date(time);
+ back.setSeconds(benchmark.getSeconds() - seconds)
+
+ return back.getTime();
+ }
+
+ private getTimeframe() {
+ const coordinatorNames = this.getCoordinatorNames();
+ let minTime = 0;
+ let maxTime = 0;
+
+ coordinatorNames.forEach(c => {
+ const coordinatorEvents = this.statistics.events.filter(e => e.coordinatorName === c);
+
+ for (const event of coordinatorEvents) {
+ for (const stage of event.stages) {
+ let startTime = new Date(stage.startTime).getTime();
+ let endTime = new Date(stage.endTime).getTime();
+
+ if (startTime < minTime) {
+ minTime = startTime;
+ }
+
+ if (endTime > maxTime) {
+ maxTime = endTime;
+ }
+ }
+ }
+ })
+
+ return [minTime, maxTime];
+ }
+
+ private getCoordinatorNames() {
+ let names: string[] = [];
+
+ this.statistics.events.forEach(e => {
+ if (!names.find(n => n === e.coordinatorName)) {
+ names.push(e.coordinatorName);
+ }
+ })
+
+ return names;
+ }
+
+ private getColor(operation: string) {
+
+ let hash = 0
+ for (let x = 0; x < operation.length; x++) {
+ let ch = operation.charCodeAt(x);
+ hash = ((hash <<5) - hash) + ch;
+ hash = hash & hash;
+ }
+
+ let r = (hash & 0xFF0000) >> 16;
+ let g = (hash & 0x00FF00) >> 8;
+ let b = hash & 0x0000FF;
+
+ return `rgb(${r}, ${g}, ${b}, 0.8)`;
+ }
+
+ private updateEventTimeline() {
+ const coordinatorNames = this.getCoordinatorNames();
+ coordinatorNames.forEach(c => {
+
+ this.eventTimelineChart.data.datasets = [];
+ this.eventTimelineChart.data.labels = [coordinatorNames];
+
+ let coordinatorEvents = this.statistics.events.filter(e => e.coordinatorName === c);
+
+ let stageStack: EventStage[] = [];
+
+ for (let eventIndex = 0; eventIndex < coordinatorEvents.length; eventIndex++) {
+ const event = coordinatorEvents[eventIndex];
+
+ if (event.stages.length > 1) {
+ for (let stageIndex = 1; stageIndex < event.stages.length; stageIndex++) {
+ let currentStage = stageStack.pop();
+ if (!currentStage) {
+ currentStage = event.stages[stageIndex - 1];
+ }
+ let nextStage = event.stages[stageIndex];
+ stageStack.push(nextStage);
+
+ this.eventTimelineChart.data.datasets.push({
+ type: 'bar',
+ label: currentStage.operation,
+ backgroundColor: this.getColor(currentStage.operation),
+ data: [new Date(currentStage.endTime).getTime() - new Date(currentStage.startTime).getTime()]
+ });
+
+ this.placeIntermediateBars(currentStage, nextStage);
+ }
+ } else {
+ stageStack.push(event.stages[0]);
+ }
+
+ const lastStage = stageStack.pop()!;
+
+ this.eventTimelineChart.data.datasets.push({
+ type: 'bar',
+ label: lastStage.operation,
+ borderColor: constants.chartColors.red,
+ borderWidth: {
+ top: 0,
+ bottom: 0,
+ left: 0,
+ right: 4
+ },
+ backgroundColor: this.getColor(lastStage.operation),
+ data: [new Date(lastStage.endTime).getTime() - new Date(lastStage.startTime).getTime()]
+ });
+ }
+
+ this.eventTimelineChart.update('none');
+ })
+ }
+
+ private placeIntermediateBars(first: EventStage, second: EventStage) {
+ let firstEnd = new Date(first.endTime).getTime();
+ let secondStart = new Date(second.startTime).getTime();
+
+ let diff = secondStart - firstEnd;
+
+ if (diff > 0) {
+ this.eventTimelineChart.data.datasets.push({
+ type: 'bar',
+ label: 'Idle',
+ backgroundColor: constants.chartColors.white,
+ data: [diff]
+ });
+ } else if (diff < 0) {
+ this.eventTimelineChart.data.datasets.push({
+ type: 'bar',
+ label: 'Overlap',
+ backgroundColor: constants.chartColors.grey,
+ data: [Math.abs(diff)]
+ });
+ }
+ }
+
+ ngOnDestroy() {
+ this.stopPollingStatistics.next(null);
+ }
+
+}
diff --git a/scripts/monitoring/src/app/modules/layout/layout.component.html b/scripts/monitoring/src/app/modules/layout/layout.component.html
new file mode 100644
index 0000000000..2b426704d2
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/layout/layout.component.html
@@ -0,0 +1,113 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<div [class.example-is-mobile]="mobileQuery.matches" class="navbar-container">
+
+ <!-- Top navbar -->
+ <mat-toolbar class="navbar" color="primary">
+
+ <button (click)="snav.toggle()" mat-icon-button>
+ <mat-icon>menu</mat-icon>
+ </button>
+
+ <a [routerLink]="['/']" class="navbar-brand" matTooltip="Home">
+ <h1>
+ Monitoring
+ </h1>
+ </a>
+
+ <span class="navbar-spacer"></span>
+
+ <button id="register-entity" [matMenuTriggerFor]="addItemMenu" color="warn" color="accent" mat-fab>
+ <mat-icon>add</mat-icon>
+ </button>
+
+ <mat-menu #addItemMenu [overlapTrigger]="false" xPosition="before" yPosition="above">
+
+ <a (click)="openNewEntityDialog('coordinator')" mat-menu-item>
+ <span>Add new coordinator</span>
+ </a>
+ <a (click)="openNewEntityDialog('worker')" mat-menu-item>
+ <span>Add new worker</span>
+ </a>
+
+ </mat-menu>
+
+
+ </mat-toolbar>
+
+ <mat-sidenav-container class="navbar-sidenav-container">
+ <!-- Side nav -->
+ <mat-sidenav #snav [fixedInViewport]="mobileQuery.matches" [mode]="mobileQuery.matches ? 'over' : 'side'"
+ [opened]="!mobileQuery.matches" class="sidenav" fixedTopGap="56">
+
+ <mat-nav-list id="menu-elements">
+ <h3 mat-subheader>Home</h3>
+
+ <a [routerLink]="['/dashboard']" mat-list-item routerLinkActive="active">
+ <mat-icon mat-list-icon>
+ dashboard
+ </mat-icon>
+ <p mat-line> Dashboard </p>
+ </a>
+
+ <a [routerLink]="['/coordinators']" mat-list-item routerLinkActive="active">
+ <mat-icon mat-list-icon>
+ supervisor_account
+ </mat-icon>
+ <p mat-line> Coordinators </p>
+ </a>
+
+ <a [routerLink]="['/workers']" mat-list-item routerLinkActive="active">
+ <mat-icon mat-list-icon>
+ engineering
+ </mat-icon>
+ <p mat-line> Workers </p>
+ </a>
+
+ <a [routerLink]="['/events']" mat-list-item routerLinkActive="active">
+ <mat-icon mat-list-icon>
+ timeline
+ </mat-icon>
+ <p mat-line> Event timeline </p>
+ </a>
+
+ <mat-divider></mat-divider>
+
+ <a href="https://systemds.apache.org/" id="push-bottom" mat-list-item routerLinkActive="active"
+ target="_blank">
+ <mat-icon mat-list-icon>
+ info_outline
+ </mat-icon>
+ <p mat-line> About </p>
+ </a>
+ </mat-nav-list>
+
+ </mat-sidenav>
+
+ <!-- Main content -->
+ <mat-sidenav-content class="sidenav-content">
+
+
+ <router-outlet></router-outlet>
+
+ </mat-sidenav-content>
+ </mat-sidenav-container>
+
+</div>
diff --git a/scripts/monitoring/src/app/modules/layout/layout.component.scss b/scripts/monitoring/src/app/modules/layout/layout.component.scss
new file mode 100644
index 0000000000..5c623f0271
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/layout/layout.component.scss
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+.navbar-spacer {
+ flex: 1 1 auto;
+}
+
+.navbar {
+ z-index: 2;
+}
+
+.navbar-brand {
+ text-decoration: none;
+ color: white;
+}
+
+.navbar-container {
+ display: flex;
+ flex-direction: column;
+ position: absolute;
+ top: 0;
+ bottom: 0;
+ left: 0;
+ right: 0;
+}
+
+.navbar-is-mobile .navbar {
+ position: fixed;
+ /* Make sure the toolbar will stay on top of the content as it scrolls past. */
+ z-index: 2;
+}
+
+.navbar-sidenav-container {
+ /* When the sidenav is not fixed, stretch the sidenav container to fill the available space. This
+ causes `<mat-sidenav-content>` to act as our scrolling element for desktop layouts. */
+ flex: 1;
+}
+
+.navbar-is-mobile .navbar-sidenav-container {
+ /* When the sidenav is fixed, don't constrain the height of the sidenav container. This allows the
+ `<body>` to be our scrolling element for mobile layouts. */
+ flex: 1 0 auto;
+}
+
+/*Set sidenav width*/
+
+mat-sidenav {
+ min-width: 180px !important;
+ border-right: 1px solid #eee;
+ box-shadow: 6px 0 6px rgba(0, 0, 0, .1);
+ /* background-color:rgb(63, 81, 181); */
+}
+
+/* Set height of wrapper to stop content from moving up & down */
+
+.progress-bar-container {
+ height: 5px;
+}
+
+a.mat-list-item.active {
+ background: rgba(0, 0, 0, .04);
+}
+
+#push-bottom {
+ position: absolute;
+ bottom: 0;
+}
diff --git a/scripts/monitoring/src/app/modules/layout/layout.component.spec.ts b/scripts/monitoring/src/app/modules/layout/layout.component.spec.ts
new file mode 100644
index 0000000000..d18a1128b0
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/layout/layout.component.spec.ts
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { LayoutComponent } from './layout.component';
+import { MatDialog } from "@angular/material/dialog";
+import { ChangeDetectorRef, DebugElement } from "@angular/core";
+import { MediaMatcher } from "@angular/cdk/layout";
+import { By } from "@angular/platform-browser";
+
+describe('LayoutComponent', () => {
+ let component: LayoutComponent;
+ let fixture: ComponentFixture<LayoutComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [LayoutComponent],
+ providers: [
+ { provide: MatDialog, useValue: {} },
+ { provide: ChangeDetectorRef, useValue: {} },
+ {
+ provide: MediaMatcher,
+ useValue: {
+ matchMedia: () => {
+ return {
+ addListener: () => {},
+ removeListener: () => {}
+ }
+ }
+ }
+ }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(LayoutComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should contain dashboard, coordinators and workers menu elements', () => {
+ let html = de.query(By.css('#menu-elements')).nativeElement.innerText;
+
+ expect(html).not.toBeNull();
+ expect(html).toContain('Dashboard');
+ expect(html).toContain('Coordinators');
+ expect(html).toContain('Workers');
+ });
+
+ it('should contain register entity button', () => {
+ expect(de.query(By.css('#register-entity'))).not.toBeNull();
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/layout/layout.component.ts b/scripts/monitoring/src/app/modules/layout/layout.component.ts
new file mode 100644
index 0000000000..945b9ae2e2
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/layout/layout.component.ts
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, OnInit, ChangeDetectorRef, OnDestroy, AfterViewInit } from '@angular/core';
+import { MediaMatcher } from '@angular/cdk/layout';
+import { MatDialog } from "@angular/material/dialog";
+import { CreateEditCoordinatorsComponent } from "../coordinators/create-edit/create-edit.component";
+import { CreateEditWorkersComponent } from "../workers/create-edit/create-edit.component";
+
+@Component({
+ selector: 'app-layout',
+ templateUrl: './layout.component.html',
+ styleUrls: ['./layout.component.scss']
+})
+export class LayoutComponent implements OnInit, OnDestroy, AfterViewInit {
+
+ mobileQuery: MediaQueryList;
+ private _mobileQueryListener: () => void;
+
+ constructor(public dialog: MatDialog,
+ private changeDetectorRef: ChangeDetectorRef,
+ private media: MediaMatcher) {
+
+ this.mobileQuery = this.media.matchMedia('(max-width: 1000px)');
+ this._mobileQueryListener = () => changeDetectorRef.detectChanges();
+ // tslint:disable-next-line: deprecation
+ this.mobileQuery.addListener(this._mobileQueryListener);
+ }
+
+ ngOnInit(): void {
+ }
+
+ ngOnDestroy(): void {
+ // tslint:disable-next-line: deprecation
+ this.mobileQuery.removeListener(this._mobileQueryListener);
+ }
+
+ ngAfterViewInit(): void {
+ this.changeDetectorRef.detectChanges();
+ }
+
+ openNewEntityDialog(type: 'worker' | 'coordinator'): void {
+
+ if (type === 'worker') {
+ this.dialog.open(CreateEditWorkersComponent, {
+ width: '500px',
+ data: null
+ });
+ } else {
+ this.dialog.open(CreateEditCoordinatorsComponent, {
+ width: '500px',
+ data: null
+ });
+ }
+ }
+}
diff --git a/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.html b/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.html
new file mode 100644
index 0000000000..6b523f8fb0
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.html
@@ -0,0 +1,38 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<h1 mat-dialog-title>Register new worker</h1>
+<div id="register-worker-content" mat-dialog-content>
+
+ <mat-form-field appearance="fill">
+ <mat-label>Name</mat-label>
+ <input *ngIf="model" [(ngModel)]="model.name" matInput>
+ </mat-form-field>
+
+ <mat-form-field appearance="fill">
+ <mat-label>Address</mat-label>
+ <input *ngIf="model" [(ngModel)]="model.address" matInput>
+ </mat-form-field>
+
+
+</div>
+<div mat-dialog-actions>
+ <button (click)="onCancelClick()" mat-button>Cancel</button>
+ <button (click)="onSaveClick()" mat-button>Save</button>
+</div>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java b/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.scss
similarity index 87%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
copy to scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.scss
index 41cf507696..6e8866c7c2 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
+++ b/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.scss
@@ -17,6 +17,6 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-
-public abstract class BaseEntityModel { }
+.mat-form-field {
+ display: block;
+}
diff --git a/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.spec.ts b/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.spec.ts
new file mode 100644
index 0000000000..9761b869d5
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.spec.ts
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CreateEditWorkersComponent } from './create-edit.component';
+import { By } from "@angular/platform-browser";
+import { DebugElement } from "@angular/core";
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { FederatedSiteServiceStub } from "../../../services/federatedSiteService.stub";
+import { MAT_DIALOG_DATA, MatDialogRef } from "@angular/material/dialog";
+
+describe('CreateEditWorkersComponent', () => {
+ let component: CreateEditWorkersComponent;
+ let fixture: ComponentFixture<CreateEditWorkersComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [CreateEditWorkersComponent],
+ providers: [
+ { provide: FederatedSiteService , useClass: FederatedSiteServiceStub },
+ { provide : MAT_DIALOG_DATA, useValue : {} },
+ { provide: MatDialogRef, useValue: {} }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(CreateEditWorkersComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should not contain null model', () => {
+ expect(component.model).not.toBeNull();
+ });
+
+ it('should contain name and address fields', () => {
+ let html = de.query(By.css('#register-worker-content')).nativeElement.innerText;
+ expect(html).toContain('Name');
+ expect(html).toContain('Address');
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.ts b/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.ts
new file mode 100644
index 0000000000..f617898a00
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/create-edit/create-edit.component.ts
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, Inject } from '@angular/core';
+import { Worker } from "../../../models/worker.model";
+import { MAT_DIALOG_DATA, MatDialogRef } from "@angular/material/dialog";
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+
+@Component({
+ selector: 'app-create-edit-worker',
+ templateUrl: './create-edit.component.html',
+ styleUrls: ['./create-edit.component.scss']
+})
+export class CreateEditWorkersComponent {
+
+ public model: Worker;
+
+ constructor(
+ private fedSiteService: FederatedSiteService,
+ public dialogRef: MatDialogRef<CreateEditWorkersComponent>,
+ @Inject(MAT_DIALOG_DATA) public id: number) {
+ }
+
+ ngOnInit(): void {
+ this.model = new Worker();
+
+ if (this.id !== null) {
+ this.fedSiteService.getWorker(this.id).subscribe(worker => this.model = worker);
+ }
+ }
+
+ onSaveClick() {
+
+ if (this.id !== null) {
+ this.fedSiteService.editWorker(this.model).subscribe(worker => {
+ this.model = worker;
+ });
+ } else {
+ this.fedSiteService.createWorker(this.model).subscribe(worker => {
+ this.model = worker;
+ });
+ }
+
+ this.dialogRef.close()
+ }
+
+ onCancelClick() {
+ this.dialogRef.close()
+ }
+}
diff --git a/scripts/monitoring/src/app/modules/workers/list/list.component.html b/scripts/monitoring/src/app/modules/workers/list/list.component.html
new file mode 100644
index 0000000000..490460ee99
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/list/list.component.html
@@ -0,0 +1,80 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<div class="container" fxLayout="row" fxLayoutAlign="center none">
+ <div fxFlex="95%">
+
+ <mat-card>
+ <mat-card-header>
+ <h2>Workers</h2>
+ <button [ngClass]="[ loadingData ? 'loading' : '']" (click)="refreshData()" color="warn" mat-mini-fab>
+ <mat-icon aria-hidden="false" aria-label="refresh">refresh</mat-icon>
+ </button>
+ </mat-card-header>
+ <mat-card-content>
+
+ <table [dataSource]="dataSource" mat-table matSort>
+
+ <ng-container matColumnDef="name">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Name</th>
+ <td *matCellDef="let element" mat-cell> {{element.name}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="address">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Address</th>
+ <td *matCellDef="let element" mat-cell> {{element.address}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="status">
+ <th *matHeaderCellDef mat-header-cell> Status</th>
+ <td *matCellDef="let element" mat-cell>
+ <span *ngIf="element.isOnline" class="online-worker">
+ Online
+ </span>
+ <span *ngIf="!element.isOnline" class="offline-worker">
+ Offline
+ </span>
+ </td>
+ </ng-container>
+
+ <ng-container matColumnDef="actions">
+ <th *matHeaderCellDef mat-header-cell> Actions</th>
+ <td *matCellDef="let element" mat-cell>
+ <button (click)="viewWorker(element.id)" color="primary" mat-mini-fab>
+ <mat-icon aria-hidden="false" aria-label="visibility">visibility</mat-icon>
+ </button>
+ <button (click)="editWorker(element.id)" color="accent" mat-mini-fab>
+ <mat-icon aria-hidden="false" aria-label="edit">edit</mat-icon>
+ </button>
+ <button (click)="deleteWorker(element.id)" color="warn" mat-mini-fab>
+ <mat-icon aria-hidden="false" aria-label="delete">delete</mat-icon>
+ </button>
+ </td>
+ </ng-container>
+
+ <tr *matHeaderRowDef="displayedColumns" mat-header-row></tr>
+ <tr *matRowDef="let row; columns: displayedColumns;" mat-row></tr>
+
+ </table>
+
+ </mat-card-content>
+ </mat-card>
+
+ </div>
+</div>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java b/scripts/monitoring/src/app/modules/workers/list/list.component.scss
similarity index 63%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
copy to scripts/monitoring/src/app/modules/workers/list/list.component.scss
index 21d6812644..7a82040134 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
+++ b/scripts/monitoring/src/app/modules/workers/list/list.component.scss
@@ -17,27 +17,51 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
+table {
+ width: 100%;
+}
-import io.netty.handler.codec.http.HttpRequest;
+th.mat-sort-header-sorted {
+ color: black;
+}
-public class Request {
- private HttpRequest _context;
- private String _body;
+mat-card-header {
+ display: flex;
- public HttpRequest getContext() {
- return _context;
+ button {
+ margin-left: auto;
}
+}
- public void setContext(final HttpRequest requestContext) {
- this._context = requestContext;
- }
+.container {
+ margin: 2em;
+}
- public String getBody() {
- return _body;
- }
+.demo-table {
+ width: 100%;
+}
- public void setBody(final String content) {
- this._body = content;
+button {
+ margin: 0.5em 0.5em 0.5em 0;
+}
+
+.online-worker {
+ color: green;
+}
+
+.offline-worker {
+ color: darkred;
+}
+
+.loading {
+ animation: rotation 2s infinite linear;
+ filter: brightness(1.5);
+}
+@keyframes rotation {
+ from {
+ transform: rotate(0deg);
+ }
+ to {
+ transform: rotate(359deg);
}
}
diff --git a/scripts/monitoring/src/app/modules/workers/list/list.component.spec.ts b/scripts/monitoring/src/app/modules/workers/list/list.component.spec.ts
new file mode 100644
index 0000000000..97e279faca
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/list/list.component.spec.ts
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ListWorkersComponent } from './list.component';
+import { DebugElement } from "@angular/core";
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { FederatedSiteServiceStub } from "../../../services/federatedSiteService.stub";
+import { Router } from "@angular/router";
+import { MatDialog } from "@angular/material/dialog";
+import { By } from "@angular/platform-browser";
+
+describe('ListWorkersComponent', () => {
+ let component: ListWorkersComponent;
+ let fixture: ComponentFixture<ListWorkersComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [ListWorkersComponent],
+ providers: [
+ { provide: FederatedSiteService , useClass: FederatedSiteServiceStub },
+ { provide : Router, useValue : {} },
+ { provide: MatDialog, useValue: {} }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(ListWorkersComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should contain table of workers', () => {
+ expect(de.query(By.css('table'))).not.toBeNull();
+ });
+
+ it('should contain name, address, status and actions table fields', () => {
+ expect(component.displayedColumns).toContain('name');
+ expect(component.displayedColumns).toContain('address');
+ expect(component.displayedColumns).toContain('status');
+ expect(component.displayedColumns).toContain('actions');
+ });
+
+ it('should not have null data source', () => {
+ expect(component.dataSource).not.toBeNull();
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/workers/list/list.component.ts b/scripts/monitoring/src/app/modules/workers/list/list.component.ts
new file mode 100644
index 0000000000..3a74849466
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/list/list.component.ts
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, ViewChild } from '@angular/core';
+import { MatSort } from '@angular/material/sort';
+import { Router } from '@angular/router';
+
+import { Worker } from 'src/app/models/worker.model';
+import { FederatedSiteService } from 'src/app/services/federatedSiteService.service';
+import { MatTableDataSource } from "@angular/material/table";
+import { CreateEditWorkersComponent } from "../create-edit/create-edit.component";
+import { MatDialog } from "@angular/material/dialog";
+
+@Component({
+ selector: 'app-list-workers',
+ templateUrl: './list.component.html',
+ styleUrls: ['./list.component.scss']
+})
+export class ListWorkersComponent {
+
+ public displayedColumns: string[] = ['name', 'address', 'status', 'actions'];
+ public dataSource: MatTableDataSource<Worker> = new MatTableDataSource<Worker>([]);
+
+ public loadingData: boolean = false;
+
+ @ViewChild(MatSort, {static: true})
+ sort: MatSort = new MatSort;
+
+ constructor(
+ public dialog: MatDialog,
+ private fedSiteService: FederatedSiteService,
+ private router: Router) {
+ }
+
+ ngOnInit(): void {
+ this.refreshData();
+ }
+
+ viewWorker(id: number) {
+ this.router.navigate(['/workers/' + id])
+ }
+
+ editWorker(id: number) {
+ this.dialog.open(CreateEditWorkersComponent, {
+ width: '500px',
+ data: id
+ });
+ }
+
+ deleteWorker(id: number) {
+ this.fedSiteService.deleteWorker(id).subscribe(() => {
+ this.dataSource.data = this.dataSource.data.filter(w => w.id !== id)
+ });
+ }
+
+ refreshData() {
+ this.loadingData = true;
+ this.fedSiteService.getAllWorkers().subscribe(workers => {
+ this.dataSource.data = workers
+ this.loadingData = false;
+ });
+ }
+
+}
diff --git a/scripts/monitoring/src/app/modules/workers/view/view.component.html b/scripts/monitoring/src/app/modules/workers/view/view.component.html
new file mode 100644
index 0000000000..2e5e049714
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/view/view.component.html
@@ -0,0 +1,89 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<div class="metrics-cards">
+ <div id="overview">
+ <mat-card class="worker-metrics-card" id="status-metric-card">
+ <mat-card-title *ngIf="model">{{model.name}}</mat-card-title>
+ <mat-card-content id="main-worker-information">
+ <h3>Address: <span *ngIf="model">{{model.address}}</span></h3>
+ <mat-divider inset></mat-divider>
+ <h3>Status:
+ <span *ngIf="model && model.isOnline" class="online-worker">
+ Online
+ </span>
+ <span *ngIf="model && !model.isOnline" class="offline-worker">
+ Offline
+ </span>
+ </h3>
+ </mat-card-content>
+ </mat-card>
+
+ <mat-card class="worker-metrics-card" id="data-metric-card">
+
+ <mat-card-content>
+ <h2>Data objects</h2>
+
+ <table [dataSource]="dataSource" mat-table matSort>
+
+ <ng-container matColumnDef="varName">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Variable name</th>
+ <td *matCellDef="let element" mat-cell> {{element.varName}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="dataType">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Data type</th>
+ <td *matCellDef="let element" mat-cell> {{element.dataType}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="valueType">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Value type</th>
+ <td *matCellDef="let element" mat-cell> {{element['valueType']}} </td>
+ </ng-container>
+
+ <ng-container matColumnDef="size">
+ <th *matHeaderCellDef mat-header-cell mat-sort-header> Byte size</th>
+ <td *matCellDef="let element" mat-cell> {{element.size}} </td>
+ </ng-container>
+
+ <tr *matHeaderRowDef="displayedColumns" mat-header-row></tr>
+ <tr *matRowDef="let row; columns: displayedColumns;" mat-row></tr>
+
+ </table>
+ </mat-card-content>
+ </mat-card>
+ </div>
+
+
+ <mat-card class="worker-metrics-card" id="requests-metric-card">
+ <canvas id="requests-metric"></canvas>
+ </mat-card>
+
+</div>
+
+<div class="metrics-cards">
+ <mat-card class="worker-metrics-card" id="cpu-metric-card">
+ <canvas id="cpu-metric"></canvas>
+ </mat-card>
+
+ <mat-card class="worker-metrics-card" id="memory-metric-card">
+ <canvas id="memory-metric"></canvas>
+ </mat-card>
+</div>
+
diff --git a/scripts/monitoring/src/app/modules/workers/view/view.component.scss b/scripts/monitoring/src/app/modules/workers/view/view.component.scss
new file mode 100644
index 0000000000..154bc7f339
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/view/view.component.scss
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* Structure */
+.worker-container {
+ position: relative;
+ margin: 2em;
+}
+
+.worker-metrics-card {
+ margin: 2em;
+}
+
+.metrics-cards {
+ display: flex;
+
+ mat-card {
+ width: 100%;
+ }
+
+ div {
+ width: 100%;
+ }
+}
+
+table {
+ width: 100%;
+}
+
+.worker-rate-limit-reached {
+ max-width: 360px;
+ text-align: center;
+}
+
+/* Column Widths */
+.mat-column-number,
+.mat-column-state {
+ max-width: 64px;
+}
+
+.mat-column-created {
+ max-width: 124px;
+}
+
+.requests-table-title {
+ padding: 1em;
+}
+
+.online-worker {
+ color: green;
+}
+
+.offline-worker {
+ color: darkred;
+}
+
+#cpu-metric-card {
+ max-width: 57.4em;
+ margin-top: 0.5em;
+ margin-right: 0.5em;
+}
+
+#memory-metric-card {
+ max-width: 57.4em;
+ margin-top: 0.5em;
+ margin-left: 0.5em;
+}
+
+#data-metric-card {
+ margin-bottom: 0.5em;
+ margin-right: 0.5em;
+ margin-top: 1em;
+ max-height: 20em;
+ overflow: auto;
+}
+
+#status-metric-card {
+ margin-bottom: 0.5em;
+ margin-right: 0.5em;
+ max-height: 7.5em;
+}
+
+#overview {
+ max-width: 57.4em;
+ float: left;
+ margin-right: 4.7em;
+}
+
+#requests-metric-card {
+ margin-bottom: 0.5em;
+ margin-left: 0.5em;
+ max-width: 57.4em;
+}
+
diff --git a/scripts/monitoring/src/app/modules/workers/view/view.component.spec.ts b/scripts/monitoring/src/app/modules/workers/view/view.component.spec.ts
new file mode 100644
index 0000000000..f3f47969a7
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/view/view.component.spec.ts
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ViewWorkerComponent } from './view.component';
+import { FederatedSiteService } from "../../../services/federatedSiteService.service";
+import { FederatedSiteServiceStub } from "../../../services/federatedSiteService.stub";
+import { ActivatedRoute } from "@angular/router";
+import { DebugElement } from "@angular/core";
+import { By } from "@angular/platform-browser";
+
+describe('ViewWorkerComponent', () => {
+ let component: ViewWorkerComponent;
+ let fixture: ComponentFixture<ViewWorkerComponent>;
+ let de: DebugElement;
+
+ beforeEach(async () => {
+ await TestBed.configureTestingModule({
+ declarations: [ViewWorkerComponent],
+ providers: [
+ { provide: FederatedSiteService , useClass: FederatedSiteServiceStub },
+ {
+ provide : ActivatedRoute,
+ useValue : {
+ snapshot: {
+ paramMap: {
+ get: () => {
+ return { id: 1 }
+ }
+ }
+ }
+ }
+ }
+ ]
+ })
+ .compileComponents();
+ });
+
+ beforeEach(() => {
+ fixture = TestBed.createComponent(ViewWorkerComponent);
+ component = fixture.componentInstance;
+ de = fixture.debugElement;
+
+ fixture.detectChanges();
+ });
+
+ it('should create', () => {
+ expect(component).toBeTruthy();
+ });
+
+ it('should not contain null model', () => {
+ expect(component.model).not.toBeNull();
+ });
+
+ it('should contain address, status and JIT information', () => {
+ let html = de.query(By.css('#main-worker-information')).nativeElement.innerText;
+ expect(html).toContain('Address');
+ expect(html).toContain('Status');
+ expect(html).toContain('JIT');
+ });
+
+ it('should contain CPU metrics diagram', () => {
+ expect(de.query(By.css('#cpu-metric-card'))).not.toBeNull();
+ });
+
+ it('should contain memory metrics diagram', () => {
+ expect(de.query(By.css('#memory-metric-card'))).not.toBeNull();
+ });
+});
diff --git a/scripts/monitoring/src/app/modules/workers/view/view.component.ts b/scripts/monitoring/src/app/modules/workers/view/view.component.ts
new file mode 100644
index 0000000000..1e2fab1c6e
--- /dev/null
+++ b/scripts/monitoring/src/app/modules/workers/view/view.component.ts
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, ViewChild } from '@angular/core';
+import { MatPaginator } from '@angular/material/paginator';
+import { MatSort } from '@angular/material/sort';
+import { ActivatedRoute } from '@angular/router';
+import { Worker } from 'src/app/models/worker.model';
+import { FederatedSiteService } from 'src/app/services/federatedSiteService.service';
+import { Statistics } from "../../../models/statistics.model";
+import { MatTableDataSource } from "@angular/material/table";
+import { DataObject } from "../../../models/dataObject.model";
+import { Chart, registerables } from "chart.js";
+import { constants } from "../../../constants";
+import 'chartjs-adapter-moment';
+import { Subject } from 'rxjs';
+import { Utils } from "../../../utils";
+
+@Component({
+ selector: 'app-view-worker',
+ templateUrl: './view.component.html',
+ styleUrls: ['./view.component.scss']
+})
+export class ViewWorkerComponent {
+
+ public displayedColumns: string[] = ['varName', 'dataType', 'valueType', 'size'];
+ public dataSource: MatTableDataSource<DataObject> = new MatTableDataSource<DataObject>([]);
+
+ public model: Worker;
+ public statistics: Statistics;
+
+ private stopPollingWorker = new Subject<any>();
+ private stopPollingStatistics = new Subject<any>();
+
+ @ViewChild(MatPaginator) paginator: MatPaginator;
+ @ViewChild(MatSort) sort: MatSort;
+
+ constructor(
+ private fedSiteService: FederatedSiteService,
+ private router: ActivatedRoute) {
+ Chart.register(...registerables);
+ }
+
+ ngOnInit(): void {
+ const id = Number(this.router.snapshot.paramMap.get('id'));
+ this.fedSiteService.getWorker(id).subscribe(worker => {
+ this.model = worker;
+ });
+
+ this.statistics = new Statistics();
+
+ const cpuMetricEle: any = document.getElementById('cpu-metric');
+ const memoryMetricEle: any = document.getElementById('memory-metric');
+ const requestsMetricEle: any = document.getElementById('requests-metric');
+
+ let cpuChart = new Chart(cpuMetricEle.getContext('2d'), {
+ type: 'line',
+ data: {
+ datasets: [{
+ data: this.statistics.utilization.map(s => {
+ return { x: new Date(s.timestamp).getTime(), y: s.cpuUsage }
+ }),
+ tension: 0.4,
+ borderColor: constants.chartColors.blue
+ }]
+ },
+ options: {
+ responsive: true,
+ plugins: {
+ legend: {
+ display: false
+ },
+ title: {
+ display: true,
+ text: 'CPU usage %'
+ }
+ },
+ scales: {
+ x: {
+ grid: {
+ display: false
+ },
+ type: 'timeseries',
+ ticks: {
+ display: false
+ }
+ },
+ y: {
+ beginAtZero: true
+ }
+ }
+ },
+ });
+
+ let memoryChart = new Chart(memoryMetricEle.getContext('2d'), {
+ type: 'line',
+ data: {
+ datasets: [{
+ data: this.statistics.utilization.map(s => {
+ return { x: new Date(s.timestamp).getTime(), y: s.memoryUsage }
+ }),
+ tension: 0.4,
+ borderColor: constants.chartColors.red
+ }]
+ },
+ options: {
+ responsive: true,
+ plugins: {
+ legend: {
+ display: false
+ },
+ title: {
+ display: true,
+ text: 'Memory usage %'
+ }
+ },
+ scales: {
+ x: {
+ grid: {
+ display: false
+ },
+ type: 'timeseries',
+ ticks: {
+ display: false
+ }
+ },
+ y: {
+ beginAtZero: true
+ }
+ }
+ },
+ });
+
+ let requestsChart = new Chart(requestsMetricEle.getContext('2d'), {
+ type: 'bar',
+ data: {
+ labels: this.statistics.requests.map(r => r.type),
+ datasets: [{
+ data: this.statistics.requests.map(r => r.count),
+ backgroundColor: constants.chartColors.purple,
+ }]
+ },
+ options: {
+ responsive: true,
+ plugins: {
+ legend: {
+ display: false
+ },
+ title: {
+ display: true,
+ text: 'Request type count'
+ }
+ },
+ scales: {
+ y: {
+ beginAtZero: true
+ }
+ }
+ },
+ });
+
+ this.fedSiteService.getWorkerPolling(id, this.stopPollingWorker).subscribe(worker => this.model = worker);
+
+ this.fedSiteService.getStatisticsPolling(id, this.stopPollingStatistics).subscribe(stats => {
+ this.statistics = stats;
+
+ cpuChart.data.datasets.forEach((dataset) => {
+ dataset.data = [];
+ this.statistics.utilization.map(s => dataset.data.push({ x: new Date(s.timestamp).getTime(), y: s.cpuUsage }));
+ dataset.data.sort(Utils.sortTimestamp);
+ });
+
+ memoryChart.data.datasets.forEach((dataset) => {
+ dataset.data = [];
+ this.statistics.utilization.map(s => dataset.data.push({ x: new Date(s.timestamp).getTime(), y: s.memoryUsage }));
+ dataset.data.sort(Utils.sortTimestamp);
+ });
+
+ requestsChart.data.labels = this.statistics.requests.map(r => r.type);
+ requestsChart.data.datasets.forEach((dataset) => {
+ dataset.data = [];
+ this.statistics.requests.map(s => dataset.data.push(s.count));
+ });
+
+ cpuChart.update();
+ memoryChart.update();
+ requestsChart.update();
+
+ this.dataSource.data = this.statistics.dataObjects;
+ })
+ }
+
+ ngOnDestroy() {
+ this.stopPollingWorker.next(null);
+ this.stopPollingStatistics.next(null);
+ }
+}
diff --git a/scripts/monitoring/src/app/services/federatedSiteService.service.ts b/scripts/monitoring/src/app/services/federatedSiteService.service.ts
new file mode 100644
index 0000000000..8618ddca4c
--- /dev/null
+++ b/scripts/monitoring/src/app/services/federatedSiteService.service.ts
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Injectable } from '@angular/core';
+import { Observable, retry, share, Subject, switchMap, takeUntil, timer } from 'rxjs';
+import { constants } from '../constants';
+import { Coordinator } from '../models/coordinator.model';
+import { Worker } from '../models/worker.model';
+import { HttpClient } from "@angular/common/http";
+import { Statistics } from "../models/statistics.model";
+
+@Injectable({
+ providedIn: 'root'
+})
+export class FederatedSiteService {
+
+ constructor(private http: HttpClient) { }
+
+ public getAllCoordinators(): Observable<Coordinator[]> {
+ return this.http.get<Coordinator[]>(constants.uriParts.coordinators);
+ }
+
+ public getAllWorkers(): Observable<Worker[]> {
+ return this.http.get<Worker[]>(constants.uriParts.workers);
+ }
+
+ public getCoordinator(id: number): Observable<Coordinator> {
+ return this.http.get<Coordinator>(constants.uriParts.coordinators + "/" + id.toString());
+ }
+
+ public getWorker(id: number): Observable<Worker> {
+ return this.http.get<Worker>(constants.uriParts.workers + "/" + id.toString());
+ }
+
+ public getWorkerPolling(id: number, stopPolling: Subject<any>): Observable<Worker> {
+ return timer(1, 3000).pipe(
+ switchMap(() => this.getWorker(id)),
+ retry(),
+ share(),
+ takeUntil(stopPolling)
+ );
+ }
+
+ public createCoordinator(coordinator: Coordinator): Observable<Coordinator> {
+ let coordinatorModel = (({name, host, processId}) => ({name, host, processId}))(coordinator);
+
+ return this.http.post<Coordinator>(constants.uriParts.coordinators, coordinatorModel);
+ }
+
+ public createWorker(worker: Worker): Observable<Worker> {
+ let workerModel = (({name, address}) => ({name, address}))(worker);
+
+ return this.http.post<Worker>(constants.uriParts.workers, workerModel);
+ }
+
+ public editCoordinator(coordinator: Coordinator): Observable<Coordinator> {
+ let coordinatorModel = (({id, name, host, processId}) => ({id, name, host, processId}))(coordinator);
+
+ return this.http.put<Coordinator>(constants.uriParts.coordinators + "/" + coordinator.id.toString(), coordinatorModel);
+ }
+
+ public editWorker(worker: Worker): Observable<Worker> {
+ let workerModel = (({id, name, address}) => ({id, name, address}))(worker);
+
+ return this.http.put<Worker>(constants.uriParts.workers + "/" + worker.id.toString(), workerModel);
+ }
+
+ public deleteCoordinator(id: number): Observable<Object> {
+ return this.http.delete(constants.uriParts.coordinators + "/" + id.toString());
+ }
+
+ public deleteWorker(id: number): Observable<Object> {
+ return this.http.delete(constants.uriParts.workers + "/" + id.toString());
+ }
+
+ public getStatistics(workerId: number): Observable<Statistics> {
+ return this.http.get<Statistics>(constants.uriParts.statistics + "/" + workerId.toString());
+ }
+
+ public getStatisticsPolling(workerId: number, stopPolling: Subject<any>): Observable<Statistics> {
+ return timer(1, 3000).pipe(
+ switchMap(() => this.getStatistics(workerId)),
+ retry(),
+ share(),
+ takeUntil(stopPolling)
+ );
+ }
+}
diff --git a/scripts/monitoring/src/app/services/federatedSiteService.stub.ts b/scripts/monitoring/src/app/services/federatedSiteService.stub.ts
new file mode 100644
index 0000000000..e0ff391d0f
--- /dev/null
+++ b/scripts/monitoring/src/app/services/federatedSiteService.stub.ts
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Observable, of } from 'rxjs';
+import { Coordinator } from '../models/coordinator.model';
+import { Worker } from '../models/worker.model';
+import { serviceMockData } from "./service-mock-data";
+import { constants } from "../constants";
+
+export class FederatedSiteServiceStub {
+
+ public getAllCoordinators() {
+ return of(serviceMockData.coordinators);
+ }
+
+ public loadCoordinators() {
+ return of(serviceMockData.coordinators);
+ }
+
+ public loadWorkers() {
+ return of(serviceMockData.workers);
+ }
+
+ public getAllWorkers() {
+ return of(serviceMockData.workers)
+ }
+
+ public getCoordinator(id: number){
+ return of(serviceMockData.coordinators.find(c => c.id === id));
+ }
+
+ public getWorker(id: number) {
+ return of(serviceMockData.workers.find(w => w.id === id));
+ }
+
+ public createCoordinator(coordinator: Coordinator) {
+ return of(coordinator)
+ }
+
+ public createWorker(worker: Worker) {
+ return of(worker);
+ }
+
+ public editCoordinator(coordinator: Coordinator) {
+ return of({"id": 42});
+ }
+
+ public editWorker(worker: Worker) {
+ return of({"id": 42});
+ }
+
+ public deleteCoordinator(id: number) {
+ return of({"id": id});
+ }
+
+ public deleteWorker(id: number) {
+ return of({"id": id});
+ }
+}
diff --git a/scripts/monitoring/src/app/services/service-mock-data.ts b/scripts/monitoring/src/app/services/service-mock-data.ts
new file mode 100644
index 0000000000..039ee04774
--- /dev/null
+++ b/scripts/monitoring/src/app/services/service-mock-data.ts
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+let coordinators = [
+ {
+ "id": 1,
+ "name": "Coordinator 1",
+ "address": "localhost:8445"
+ },
+ {
+ "id": 2,
+ "name": "Coordinator 1",
+ "address": "localhost:8446"
+ }
+]
+
+let workers = [
+ {
+ "id": 1,
+ "name": "Worker 1",
+ "address": "localhost:8001",
+ "isOnline": false,
+ "jitCompileTime": 0,
+ "requestTypeCounts": [
+ {"type": "GET_VAR", "count": 0},
+ {"type": "PUT_VAR", "count": 0},
+ {"type": "READ_VAR", "count": 0},
+ {"type": "EXEC_UDF", "count": 0},
+ {"type": "EXEC_INST", "count": 0}
+ ],
+ "stats": []
+ },
+ {
+ "id": 2,
+ "name": "Worker 2",
+ "address": "localhost:8002",
+ "isOnline": false,
+ "jitCompileTime": 0,
+ "requestTypeCounts": [
+ {"type": "GET_VAR", "count": 0},
+ {"type": "PUT_VAR", "count": 0},
+ {"type": "READ_VAR", "count": 0},
+ {"type": "EXEC_UDF", "count": 0},
+ {"type": "EXEC_INST", "count": 0}
+ ],
+ "stats": [{
+ "timestamp": "2022-06-25 13:18:19.578",
+ "x": 1.96,
+ "memoryUsage": 1.46,
+ "coordinatorTraffic": [],
+ "heavyHitters": []
+ }, {
+ "timestamp": "2022-06-25 13:18:22.522",
+ "x": 1.95,
+ "memoryUsage": 1.46,
+ "coordinatorTraffic": [],
+ "heavyHitters": []
+ }]
+ },
+ {
+ "id": 3,
+ "name": "Worker 3",
+ "address": "localhost:8003",
+ "isOnline": true,
+ "jitCompileTime": 3.69,
+ "requestTypeCounts": [
+ {"type": "GET_VAR", "count": 0},
+ {"type": "PUT_VAR", "count": 0},
+ {"type": "READ_VAR", "count": 0},
+ {"type": "EXEC_UDF", "count": 0},
+ {"type": "EXEC_INST", "count": 0}
+ ],
+ "stats": [{
+ "timestamp": "2022-06-25 13:18:19.578",
+ "x": 1.96,
+ "memoryUsage": 1.46,
+ "coordinatorTraffic": [],
+ "heavyHitters": []
+ }, {
+ "timestamp": "2022-06-25 13:18:22.522",
+ "x": 1.95,
+ "memoryUsage": 1.46,
+ "coordinatorTraffic": [],
+ "heavyHitters": []
+ }]
+ }
+]
+
+export const serviceMockData = {
+ workers: workers,
+ coordinators: coordinators
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/app/utils.ts
similarity index 78%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/app/utils.ts
index 18b17ea7fc..d16a6ac684 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/app/utils.ts
@@ -17,10 +17,12 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+export class Utils {
+ public static sortTimestamp(a, b) {
+ return a.x < b.x ? -1 : (a.x > b.x ? 1 : 0);
+ }
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+ public static sortStartDate(a, b) {
+ return a.startTime < b.startTime ? -1 : (a.startTime > b.startTime ? 1 : 0);
+ }
}
diff --git a/scripts/monitoring/src/assets/favicon.png b/scripts/monitoring/src/assets/favicon.png
new file mode 100644
index 0000000000..c5311b994e
Binary files /dev/null and b/scripts/monitoring/src/assets/favicon.png differ
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java b/scripts/monitoring/src/environments/environment.prod.ts
similarity index 87%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
copy to scripts/monitoring/src/environments/environment.prod.ts
index 41cf507696..25b168b721 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
+++ b/scripts/monitoring/src/environments/environment.prod.ts
@@ -17,6 +17,6 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-
-public abstract class BaseEntityModel { }
+export const environment = {
+ production: true
+};
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java b/scripts/monitoring/src/environments/environment.ts
similarity index 55%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
copy to scripts/monitoring/src/environments/environment.ts
index dd683080e2..ec3000bbd5 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
+++ b/scripts/monitoring/src/environments/environment.ts
@@ -17,22 +17,19 @@
* under the License.
*/
+// This file can be replaced during build by using the `fileReplacements` array.
+// `ng build` replaces `environment.ts` with `environment.prod.ts`.
+// The list of file replacements can be found in `angular.json`.
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+export const environment = {
+ production: false
+};
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
-
-import java.util.List;
-
-public interface IRepository {
- Long createEntity(EntityEnum type, BaseEntityModel model);
-
- BaseEntityModel getEntity(EntityEnum type, Long id);
-
- List<BaseEntityModel> getAllEntities(EntityEnum type);
-
- List<BaseEntityModel> getAllEntitiesByField(EntityEnum type, Object fieldValue);
- void updateEntity(EntityEnum type, BaseEntityModel model);
-
- void removeEntity(EntityEnum type, Long id);
-}
+/*
+ * For easier debugging in development mode, you can import the following file
+ * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`.
+ *
+ * This import should be commented out in production mode because it will have a negative impact
+ * on performance if an error is thrown.
+ */
+// import 'zone.js/plugins/zone-error'; // Included with Angular CLI.
diff --git a/scripts/monitoring/src/index.html b/scripts/monitoring/src/index.html
new file mode 100644
index 0000000000..f0014f8fbc
--- /dev/null
+++ b/scripts/monitoring/src/index.html
@@ -0,0 +1,35 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!doctype html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title>MonitoringUi</title>
+ <base href="/">
+ <meta content="width=device-width, initial-scale=1" name="viewport">
+ <link href="assets/favicon.png" rel="icon" type="image/png">
+ <link href="https://fonts.gstatic.com" rel="preconnect">
+ <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500&display=swap" rel="stylesheet">
+ <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
+</head>
+<body class="mat-typography">
+<app-root></app-root>
+</body>
+</html>
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/main.ts
similarity index 68%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/main.ts
index 18b17ea7fc..2a31e1ecb5 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/main.ts
@@ -17,10 +17,15 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+import { enableProdMode } from '@angular/core';
+import { platformBrowserDynamic } from '@angular/platform-browser-dynamic';
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+import { AppModule } from './app/app.module';
+import { environment } from './environments/environment';
+
+if (environment.production) {
+ enableProdMode();
}
+
+platformBrowserDynamic().bootstrapModule(AppModule)
+ .catch(err => console.error(err));
diff --git a/scripts/monitoring/src/polyfills.ts b/scripts/monitoring/src/polyfills.ts
new file mode 100644
index 0000000000..5f3fc132f2
--- /dev/null
+++ b/scripts/monitoring/src/polyfills.ts
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * This file includes polyfills needed by Angular and is loaded before the app.
+ * You can add your own extra polyfills to this file.
+ *
+ * This file is divided into 2 sections:
+ * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers.
+ * 2. Application imports. Files imported after ZoneJS that should be loaded before your main
+ * file.
+ *
+ * The current setup is for so-called "evergreen" browsers; the last versions of browsers that
+ * automatically update themselves. This includes recent versions of Safari, Chrome (including
+ * Opera), Edge on the desktop, and iOS and Chrome on mobile.
+ *
+ * Learn more in https://angular.io/guide/browser-support
+ */
+
+/***************************************************************************************************
+ * BROWSER POLYFILLS
+ */
+
+/**
+ * By default, zone.js will patch all possible macroTask and DomEvents
+ * user can disable parts of macroTask/DomEvents patch by setting following flags
+ * because those flags need to be set before `zone.js` being loaded, and webpack
+ * will put import in the top of bundle, so user need to create a separate file
+ * in this directory (for example: zone-flags.ts), and put the following flags
+ * into that file, and then add the following code before importing zone.js.
+ * import './zone-flags';
+ *
+ * The flags allowed in zone-flags.ts are listed here.
+ *
+ * The following flags will work for all browsers.
+ *
+ * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame
+ * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick
+ * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames
+ *
+ * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js
+ * with the following flag, it will bypass `zone.js` patch for IE/Edge
+ *
+ * (window as any).__Zone_enable_cross_context_check = true;
+ *
+ */
+
+/***************************************************************************************************
+ * Zone JS is required by default for Angular itself.
+ */
+import 'zone.js'; // Included with Angular CLI.
+
+
+/***************************************************************************************************
+ * APPLICATION IMPORTS
+ */
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/src/styles.scss
similarity index 81%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/src/styles.scss
index 18b17ea7fc..d4a1c90f4d 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/src/styles.scss
@@ -17,10 +17,13 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+/* You can add global styles to this file, and also import other style files */
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+html, body {
+ height: 100%;
+}
+
+body {
+ margin: 0;
+ font-family: Roboto, "Helvetica Neue", sans-serif;
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java b/scripts/monitoring/src/test.ts
similarity index 52%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java
copy to scripts/monitoring/src/test.ts
index 6016748bc8..e27f55640a 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java
+++ b/scripts/monitoring/src/test.ts
@@ -17,20 +17,29 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers;
-
-import io.netty.handler.codec.http.FullHttpResponse;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Request;
-
-public interface IController {
-
- FullHttpResponse create(final Request request);
-
- FullHttpResponse update(final Request request, final Long objectId);
-
- FullHttpResponse delete(final Request request, final Long objectId);
-
- FullHttpResponse get(final Request request, final Long objectId);
-
- FullHttpResponse getAll(final Request request);
-}
+// This file is required by karma.conf.js and loads recursively all the .spec and framework files
+
+import 'zone.js/testing';
+import { getTestBed } from '@angular/core/testing';
+import {
+ BrowserDynamicTestingModule,
+ platformBrowserDynamicTesting
+} from '@angular/platform-browser-dynamic/testing';
+
+declare const require: {
+ context(path: string, deep?: boolean, filter?: RegExp): {
+ <T>(id: string): T;
+ keys(): string[];
+ };
+};
+
+// First, initialize the Angular testing environment.
+getTestBed().initTestEnvironment(
+ BrowserDynamicTestingModule,
+ platformBrowserDynamicTesting(),
+);
+
+// Then we find all the tests.
+const context = require.context('./', true, /\.spec\.ts$/);
+// And load the modules.
+context.keys().map(context);
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/tsconfig.app.json
similarity index 73%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/tsconfig.app.json
index 18b17ea7fc..3ebac19efa 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/tsconfig.app.json
@@ -17,10 +17,18 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+ "extends": "./tsconfig.json",
+ "compilerOptions": {
+ "outDir": "./out-tsc/app",
+ "types": []
+ },
+ "files": [
+ "src/main.ts",
+ "src/polyfills.ts"
+ ],
+ "include": [
+ "src/**/*.d.ts"
+ ]
}
diff --git a/scripts/monitoring/tsconfig.json b/scripts/monitoring/tsconfig.json
new file mode 100644
index 0000000000..d38b9bfb9f
--- /dev/null
+++ b/scripts/monitoring/tsconfig.json
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+ "compileOnSave": false,
+ "compilerOptions": {
+ "baseUrl": "./",
+ "outDir": "./dist/out-tsc",
+ "forceConsistentCasingInFileNames": true,
+ "strict": true,
+ "noImplicitReturns": true,
+ "noImplicitAny": false,
+ "noFallthroughCasesInSwitch": true,
+ "strictPropertyInitialization": false,
+ "sourceMap": true,
+ "skipLibCheck": true,
+ "resolveJsonModule": true,
+ "esModuleInterop": true,
+ "declaration": false,
+ "downlevelIteration": true,
+ "experimentalDecorators": true,
+ "moduleResolution": "node",
+ "importHelpers": true,
+ "target": "es2017",
+ "module": "es2020",
+ "lib": [
+ "es2018",
+ "dom"
+ ],
+ "types": ["jest", "node"]
+ },
+ "angularCompilerOptions": {
+ "enableI18nLegacyMessageIdFormat": false,
+ "strictInjectionParameters": true,
+ "strictInputAccessModifiers": true,
+ "strictTemplates": true
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/scripts/monitoring/tsconfig.spec.json
similarity index 71%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
copy to scripts/monitoring/tsconfig.spec.json
index 18b17ea7fc..d8b2d47480 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/scripts/monitoring/tsconfig.spec.json
@@ -17,10 +17,21 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+ "extends": "./tsconfig.json",
+ "compilerOptions": {
+ "outDir": "./out-tsc/spec",
+ "types": [
+ "jasmine"
+ ]
+ },
+ "files": [
+ "src/test.ts",
+ "src/polyfills.ts"
+ ],
+ "include": [
+ "src/**/*.spec.ts",
+ "src/**/*.d.ts"
+ ]
}
diff --git a/src/main/java/org/apache/sysds/api/DMLOptions.java b/src/main/java/org/apache/sysds/api/DMLOptions.java
index 11bcea1604..d9bbe260bc 100644
--- a/src/main/java/org/apache/sysds/api/DMLOptions.java
+++ b/src/main/java/org/apache/sysds/api/DMLOptions.java
@@ -198,6 +198,7 @@ public class DMLOptions {
else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime]");
}
}
+
dmlOptions.stats = line.hasOption("stats");
if (dmlOptions.stats){
String statsCount = line.getOptionValue("stats");
@@ -348,6 +349,9 @@ public class DMLOptions {
Option pythonOpt = OptionBuilder
.withDescription("Python Context start with port argument for communication to python")
.isRequired().hasArg().create("python");
+ Option monitorIdOpt = OptionBuilder
+ .withDescription("Coordinator context start with monitorId argument for monitoring registration")
+ .hasOptionalArg().create("monitorId");
Option fileOpt = OptionBuilder.withArgName("filename")
.withDescription("specifies dml/pydml file to execute; path can be local/hdfs/gpfs (prefixed with appropriate URI)")
.isRequired().hasArg().create("f");
@@ -393,6 +397,7 @@ public class DMLOptions {
options.addOption(lineageOpt);
options.addOption(fedOpt);
options.addOption(monitorOpt);
+ options.addOption(monitorIdOpt);
options.addOption(checkPrivacy);
options.addOption(federatedCompilation);
options.addOption(noFedRuntimeConversion);
diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java
index 9f6eb656e0..9f0ab9dfb7 100644
--- a/src/main/java/org/apache/sysds/api/DMLScript.java
+++ b/src/main/java/org/apache/sysds/api/DMLScript.java
@@ -260,7 +260,7 @@ public class DMLScript
LINEAGE_ESTIMATE = dmlOptions.lineage_estimate;
CHECK_PRIVACY = dmlOptions.checkPrivacy;
LINEAGE_DEBUGGER = dmlOptions.lineage_debugger;
- SEED = dmlOptions.seed;
+ SEED = dmlOptions.seed;
String fnameOptConfig = dmlOptions.configFile;
boolean isFile = dmlOptions.filePath != null;
@@ -408,7 +408,7 @@ public class DMLScript
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs)
throws IOException
{
- //print basic time and environment info
+ //print basic time environment info and process id
printStartExecInfo( dmlScriptStr );
//Step 1: parse configuration files & write any configuration specific global variables
@@ -578,6 +578,7 @@ public class DMLScript
private static void printStartExecInfo(String dmlScriptString) {
LOG.info("BEGIN DML run " + getDateTime());
LOG.debug("DML script: \n" + dmlScriptString);
+ LOG.info("Process id: " + IDHandler.obtainProcessID());
}
private static String getDateTime() {
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
index 370163aaf2..9497de1f2f 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
@@ -19,8 +19,11 @@
package org.apache.sysds.runtime.controlprogram.federated;
+import java.io.IOException;
import java.io.Serializable;
import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -29,8 +32,10 @@ import java.util.concurrent.Future;
import javax.net.ssl.SSLException;
+import io.netty.channel.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.api.DMLScript;
import org.apache.sysds.common.Types;
import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.conf.DMLConfig;
@@ -42,12 +47,6 @@ import org.apache.sysds.runtime.meta.MetaData;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInboundHandlerAdapter;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
@@ -72,6 +71,7 @@ public class FederatedData {
private final Types.DataType _dataType;
private final InetSocketAddress _address;
private final String _filepath;
+ private static final int endOfDynamicPorts = 65535;
/**
* The ID of default matrix/tensor on which operations get executed if no other ID is given.
@@ -200,6 +200,20 @@ public class FederatedData {
}
}
+ private static int getAvailablePort(int monitorId, int maxMonitorCoordinators) {
+
+ for (int i = 0; i < maxMonitorCoordinators; i++) {
+ int tmpPort = endOfDynamicPorts - monitorId - i * maxMonitorCoordinators;
+ try(ServerSocket availableSocket = new ServerSocket(tmpPort)) {
+ return availableSocket.getLocalPort();
+ }
+ catch(IOException ignored) {
+ }
+ }
+
+ return -1;
+ }
+
private static ChannelInitializer<SocketChannel> createChannel(InetSocketAddress address, DataRequestHandler handler){
final int timeout = ConfigurationManager.getFederatedTimeout();
final boolean ssl = ConfigurationManager.isFederatedSSL();
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java
index 9620648630..46d6d26e13 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java
@@ -26,7 +26,6 @@ import java.lang.management.ThreadMXBean;
import java.net.InetSocketAddress;
import java.text.DecimalFormat;
import java.time.LocalDateTime;
-import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -34,26 +33,33 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.LongAdder;
import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.ImmutableTriple;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.lang3.tuple.Triple;
import org.apache.sysds.api.DMLScript;
+import org.apache.sysds.common.Types;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.controlprogram.caching.CacheBlock;
import org.apache.sysds.runtime.controlprogram.caching.CacheStatistics;
import org.apache.sysds.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysds.runtime.controlprogram.caching.FrameObject;
+import org.apache.sysds.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType;
import org.apache.sysds.runtime.controlprogram.federated.FederatedStatistics.FedStatsCollection.CacheStatsCollection;
import org.apache.sysds.runtime.controlprogram.federated.FederatedStatistics.FedStatsCollection.GCStatsCollection;
import org.apache.sysds.runtime.controlprogram.federated.FederatedStatistics.FedStatsCollection.LineageCacheStatsCollection;
import org.apache.sysds.runtime.controlprogram.federated.FederatedStatistics.FedStatsCollection.MultiTenantStatsCollection;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.DataObjectModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.EventModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.RequestModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.TrafficModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.UtilizationModel;
import org.apache.sysds.runtime.instructions.InstructionUtils;
import org.apache.sysds.runtime.instructions.cp.Data;
import org.apache.sysds.runtime.instructions.cp.ListObject;
@@ -97,9 +103,10 @@ public class FederatedStatistics {
private static final LongAdder fedPutLineageItems = new LongAdder();
private static final LongAdder fedSerializationReuseCount = new LongAdder();
private static final LongAdder fedSerializationReuseBytes = new LongAdder();
- // Traffic between federated worker and a coordinator site
- // in the form of [{ datetime, coordinatorAddress, transferredBytes }, { ... }] }
- private static CopyOnWriteArrayList<Triple<LocalDateTime, String, Long>> coordinatorsTrafficBytes = new CopyOnWriteArrayList<>();
+ private static final List<TrafficModel> coordinatorsTrafficBytes = new ArrayList<>();
+ private static final List<EventModel> workerEvents = new ArrayList<>();
+ private static final Map<String, DataObjectModel> workerDataObjects = new HashMap<>();
+ private static final Map<String, RequestModel> workerFederatedRequests = new HashMap<>();
public static void logServerTraffic(long read, long written) {
bytesReceived.add(read);
@@ -111,7 +118,6 @@ public class FederatedStatistics {
fedBytesSent.add(written);
}
-
public static synchronized void incFederated(RequestType rqt, List<Object> data){
switch (rqt) {
case READ_VAR:
@@ -142,10 +148,10 @@ public class FederatedStatistics {
}
private static void incFedTransfer(Object dataObj) {
- incFedTransfer(dataObj, null);
+ incFedTransfer(dataObj, null, null);
}
- public static void incFedTransfer(Object dataObj, String host) {
+ public static void incFedTransfer(Object dataObj, String host, Long pid) {
long byteAmount = 0;
if(dataObj instanceof MatrixBlock) {
transferredMatrixCount.increment();
@@ -157,15 +163,28 @@ public class FederatedStatistics {
byteAmount = ((FrameBlock)dataObj).getInMemorySize();
transferredFrameBytes.add(byteAmount);
}
- else if(dataObj instanceof ScalarObject)
+ else if(dataObj instanceof ScalarObject) {
transferredScalarCount.increment();
- else if(dataObj instanceof ListObject)
+ }
+ else if(dataObj instanceof ListObject) {
transferredListCount.increment();
- else if(dataObj instanceof MatrixCharacteristics)
+ var listData = ((ListObject)dataObj).getData();
+ for (var entry: listData) {
+ if (entry.getDataType().isMatrix()) {
+ byteAmount += ((MatrixObject)entry).getDataSize();
+ } else if (entry.getDataType().isFrame()) {
+ byteAmount += ((FrameObject)entry).getDataSize();
+ }
+ }
+ }
+ else if(dataObj instanceof MatrixCharacteristics) {
transferredMatCharCount.increment();
+ }
+
+ if (host != null && pid != null) {
+ var coordinatorHostId = String.format("%s-%d", host, pid);
- if (host != null && byteAmount > 0) {
- coordinatorsTrafficBytes.add(new ImmutableTriple<>(LocalDateTime.now(), host, byteAmount));
+ coordinatorsTrafficBytes.add(new TrafficModel(LocalDateTime.now(), coordinatorHostId, byteAmount));
}
}
@@ -208,6 +227,8 @@ public class FederatedStatistics {
fedBytesReceived.reset();
//TODO merge with existing
coordinatorsTrafficBytes.clear();
+ workerEvents.clear();
+ workerDataObjects.clear();
}
public static String displayFedIOExecStatistics() {
@@ -272,12 +293,14 @@ public class FederatedStatistics {
sb.append(displayFedReuseReadStats());
sb.append(displayFedPutLineageStats());
sb.append(displayFedSerializationReuseStats());
+ sb.append(displayFedTransfer());
//FIXME: the following statistics need guards to only show
// results if federated operations where executed, also the CPU
// and mem usage only probe once at the time of stats printing
//sb.append(displayFedTransfer());
//sb.append(displayCPUUsage());
//sb.append(displayMemoryUsage());
+
return sb.toString();
}
@@ -294,8 +317,6 @@ public class FederatedStatistics {
sb.append(displayGCStats(fedStats.gcStats));
sb.append(displayLinCacheStats(fedStats.linCacheStats));
sb.append(displayMultiTenantStats(fedStats.mtStats));
- sb.append(displayCPUUsage());
- sb.append(displayMemoryUsage());
sb.append(displayFedTransfer());
sb.append(displayHeavyHitters(fedStats.heavyHitters, numHeavyHitters));
sb.append(displayNetworkTrafficStatistics());
@@ -350,33 +371,12 @@ public class FederatedStatistics {
sb.append("Transferred bytes (Host/Datetime/ByteAmount):\n");
for (var entry: coordinatorsTrafficBytes) {
- sb.append(String.format("%s/%s/%d.\n",
- entry.getLeft().format(DateTimeFormatter.ISO_DATE_TIME), entry.getMiddle(), entry.getRight()));
+ sb.append(String.format("%s/%s/%d.\n", entry.getCoordinatorHostId(), entry.timestamp, entry.byteAmount));
}
return sb.toString();
}
- private static String displayCPUUsage() {
- StringBuilder sb = new StringBuilder();
-
- double cpuUsage = getCPUUsage();
-
- sb.append(String.format("CPU usage %%: %.2f\n", cpuUsage));
-
- return sb.toString();
- }
-
- private static String displayMemoryUsage() {
- StringBuilder sb = new StringBuilder();
-
- double memoryUsage = getMemoryUsage();
-
- sb.append(String.format("Memory usage %%: %.2f\n", memoryUsage));
-
- return sb.toString();
- }
-
private static String displayHeavyHitters(HashMap<String, Pair<Long, Double>> heavyHitters, int num) {
StringBuilder sb = new StringBuilder();
@SuppressWarnings("unchecked")
@@ -479,30 +479,59 @@ public class FederatedStatistics {
return fedLookupTableGetCount.longValue();
}
- public static List<Triple<LocalDateTime, String, Long>> getCoordinatorsTrafficBytes() {
- return coordinatorsTrafficBytes;
+ public static List<TrafficModel> getCoordinatorsTrafficBytes() {
+ var result = new ArrayList<>(coordinatorsTrafficBytes);
+
+ coordinatorsTrafficBytes.clear();
+
+ return result;
}
- public static double getCPUUsage() {
- ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
- double cpuUsage = 0.0f;
+ public static List<EventModel> getWorkerEvents() {
+ var result = new ArrayList<>(workerEvents);
- for(Long threadID : threadMXBean.getAllThreadIds()) {
- cpuUsage += threadMXBean.getThreadCpuTime(threadID);
- }
+ workerEvents.clear();
+
+ return result;
+ }
+ public static List<RequestModel> getWorkerRequests() {
+ return new ArrayList<>(workerFederatedRequests.values());
+ }
- cpuUsage /= 1000000000; // nanoseconds to seconds
+ public static List<DataObjectModel> getWorkerDataObjects() {
+ return new ArrayList<>(workerDataObjects.values());
+ }
- return cpuUsage;
+ public static void addEvent(EventModel event) {
+ workerEvents.add(event);
}
+ public static void addWorkerRequest(RequestModel request) {
+ if (!workerFederatedRequests.containsKey(request.type)) {
+ workerFederatedRequests.put(request.type, request);
+ };
- public static double getMemoryUsage() {
- MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+ workerFederatedRequests.get(request.type).count++;
+ }
+ public static void addDataObject(DataObjectModel dataObject) {
+ workerDataObjects.put(dataObject.varName, dataObject);
+ }
+ public static void removeDataObjects() {
+ workerDataObjects.clear();
+ }
+
+ public static UtilizationModel getUtilization() {
+ var osMXBean = ManagementFactory.getOperatingSystemMXBean();
+ var memoryMXBean = ManagementFactory.getMemoryMXBean();
+
+ double cpuUsage = osMXBean.getSystemLoadAverage();
+ double memoryUsage = 0.0;
double maxMemory = (double)memoryMXBean.getHeapMemoryUsage().getMax() / 1073741824;
double usedMemory = (double)memoryMXBean.getHeapMemoryUsage().getUsed() / 1073741824;
- return (usedMemory / maxMemory) * 100;
+ memoryUsage = (usedMemory / maxMemory) * 100;
+
+ return new UtilizationModel(cpuUsage, memoryUsage);
}
public static long getFedLookupTableGetTime() {
@@ -654,20 +683,21 @@ public class FederatedStatistics {
private void collectStats() {
cacheStats.collectStats();
jitCompileTime = ((double)Statistics.getJITCompileTime()) / 1000; // in sec
- cpuUsage = getCPUUsage();
- memoryUsage = getMemoryUsage();
+ utilization = getUtilization();
gcStats.collectStats();
linCacheStats.collectStats();
mtStats.collectStats();
heavyHitters = Statistics.getHeavyHittersHashMap();
coordinatorsTrafficBytes = getCoordinatorsTrafficBytes();
+ workerEvents = getWorkerEvents();
+ workerDataObjects = getWorkerDataObjects();
+ workerRequests = getWorkerRequests();
}
public void aggregate(FedStatsCollection that) {
cacheStats.aggregate(that.cacheStats);
jitCompileTime += that.jitCompileTime;
- cpuUsage += that.cpuUsage;
- memoryUsage += that.memoryUsage;
+ utilization = that.utilization;
gcStats.aggregate(that.gcStats);
linCacheStats.aggregate(that.linCacheStats);
mtStats.aggregate(that.mtStats);
@@ -675,7 +705,10 @@ public class FederatedStatistics {
(key, value) -> heavyHitters.merge(key, value, (v1, v2) ->
new ImmutablePair<>(v1.getLeft() + v2.getLeft(), v1.getRight() + v2.getRight()))
);
- that.coordinatorsTrafficBytes.addAll(coordinatorsTrafficBytes);
+ coordinatorsTrafficBytes.addAll(that.coordinatorsTrafficBytes);
+ workerEvents.addAll(that.workerEvents);
+ workerDataObjects.addAll(that.workerDataObjects);
+ workerRequests.addAll(that.workerRequests);
}
protected static class CacheStatsCollection implements Serializable {
@@ -823,12 +856,15 @@ public class FederatedStatistics {
private CacheStatsCollection cacheStats = new CacheStatsCollection();
public double jitCompileTime = 0;
- public double cpuUsage = 0;
- public double memoryUsage = 0;
+ public UtilizationModel utilization = new UtilizationModel(0.0, 0.0);
private GCStatsCollection gcStats = new GCStatsCollection();
private LineageCacheStatsCollection linCacheStats = new LineageCacheStatsCollection();
private MultiTenantStatsCollection mtStats = new MultiTenantStatsCollection();
public HashMap<String, Pair<Long, Double>> heavyHitters = new HashMap<>();
- public List<Triple<LocalDateTime, String, Long>> coordinatorsTrafficBytes = new ArrayList<>();
+ public List<TrafficModel> coordinatorsTrafficBytes = new ArrayList<>();
+ public List<EventModel> workerEvents = new ArrayList<>();
+ public List<DataObjectModel> workerDataObjects = new ArrayList<>();
+
+ public List<RequestModel> workerRequests = new ArrayList<>();
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
index 509e0998eb..4ddf23a1d3 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
@@ -23,6 +23,7 @@ import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
+import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.concurrent.CompletableFuture;
@@ -49,6 +50,10 @@ import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType;
import org.apache.sysds.runtime.controlprogram.federated.FederatedResponse.ResponseType;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.DataObjectModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.EventModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.EventStageModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.RequestModel;
import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing;
import org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysds.runtime.instructions.Instruction;
@@ -189,6 +194,9 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
FederatedResponse response = null; // last response
boolean containsCLEAR = false;
long clearReqPid = -1;
+ var event = new EventModel();
+ final String coordinatorHostIdFormat = "%s-%d";
+ event.setCoordinatorHostId(String.format(coordinatorHostIdFormat, remoteHost, requests[0].getPID()));
for(int i = 0; i < requests.length; i++) {
final FederatedRequest request = requests[i];
final RequestType t = request.getType();
@@ -198,13 +206,25 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
PrivacyMonitor.setCheckPrivacy(request.checkPrivacy());
PrivacyMonitor.clearCheckedConstraints();
+ var eventStage = new EventStageModel();
// execute command and handle privacy constraints
- final FederatedResponse tmp = executeCommand(request, ecm);
+ final FederatedResponse tmp = executeCommand(request, ecm, eventStage);
+
+ if (DMLScript.STATISTICS) {
+ var requestStat = new RequestModel(request.getType().name(), 1L);
+ requestStat.setCoordinatorHostId(String.format(coordinatorHostIdFormat, remoteHost, request.getPID()));
+ FederatedStatistics.addWorkerRequest(requestStat);
+
+ event.stages.add(eventStage);
+ }
+
conditionalAddCheckedConstraints(request, tmp);
// select the response
if(!tmp.isSuccessful()) {
LOG.error("Command " + t + " resulted in error:\n" + tmp.getErrorMessage());
+ if (DMLScript.STATISTICS)
+ FederatedStatistics.addEvent(event);
return tmp; // Return first error without executing anything further
}
else if(t == RequestType.GET_VAR) {
@@ -212,6 +232,8 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
if(response != null) {
String message = "Multiple GET_VAR are not supported in single batch of requests.";
LOG.error(message);
+ if (DMLScript.STATISTICS)
+ FederatedStatistics.addEvent(event);
throw new FederatedWorkerHandlerException(message);
}
response = tmp;
@@ -220,17 +242,18 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
response = tmp; // return last
}
-
- if(t == RequestType.PUT_VAR || t == RequestType.EXEC_UDF) {
- for (int paramIndex = 0; paramIndex < request.getNumParams(); paramIndex++) {
- FederatedStatistics.incFedTransfer(request.getParam(paramIndex), _remoteAddress);
+ if (DMLScript.STATISTICS) {
+ if(t == RequestType.PUT_VAR || t == RequestType.EXEC_UDF) {
+ for (int paramIndex = 0; paramIndex < request.getNumParams(); paramIndex++) {
+ FederatedStatistics.incFedTransfer(request.getParam(paramIndex), _remoteAddress, request.getPID());
+ }
}
- }
- if(t == RequestType.GET_VAR) {
- var data = response.getData();
- for (int dataObjIndex = 0; dataObjIndex < Arrays.stream(data).count(); dataObjIndex++) {
- FederatedStatistics.incFedTransfer(data[dataObjIndex], _remoteAddress);
+ if(t == RequestType.GET_VAR) {
+ var data = response.getData();
+ for (int dataObjIndex = 0; dataObjIndex < Arrays.stream(data).count(); dataObjIndex++) {
+ FederatedStatistics.incFedTransfer(data[dataObjIndex], _remoteAddress, request.getPID());
+ }
}
}
@@ -245,6 +268,9 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
printStatistics();
}
+ if (DMLScript.STATISTICS)
+ FederatedStatistics.addEvent(event);
+
return response;
}
@@ -268,28 +294,45 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
response.setCheckedConstraints(PrivacyMonitor.getCheckedConstraints());
}
- private FederatedResponse executeCommand(FederatedRequest request, ExecutionContextMap ecm)
+ private FederatedResponse executeCommand(FederatedRequest request, ExecutionContextMap ecm, EventStageModel eventStage)
throws DMLPrivacyException, FederatedWorkerHandlerException, Exception {
final RequestType method = request.getType();
+ FederatedResponse result = null;
+
+ eventStage.startTime = LocalDateTime.now();
+
switch(method) {
case READ_VAR:
- return readData(request, ecm); // matrix/frame
+ result = readData(request, ecm); // matrix/frame
+ break;
case PUT_VAR:
- return putVariable(request, ecm);
+ eventStage.operation = method.name();
+ result = putVariable(request, ecm);
+ break;
case GET_VAR:
- return getVariable(request, ecm);
+ eventStage.operation = method.name();
+ result = getVariable(request, ecm);
+ break;
case EXEC_INST:
- return execInstruction(request, ecm);
+ result = execInstruction(request, ecm, eventStage);
+ break;
case EXEC_UDF:
- return execUDF(request, ecm);
+ result = execUDF(request, ecm);
+ break;
case CLEAR:
- return execClear(ecm);
+ result = execClear(ecm);
+ break;
case NOOP:
- return execNoop();
+ result = execNoop();
+ break;
default:
String message = String.format("Method %s is not supported.", method);
throw new FederatedWorkerHandlerException(message);
}
+
+ eventStage.endTime = LocalDateTime.now();
+
+ return result;
}
private FederatedResponse readData(FederatedRequest request, ExecutionContextMap ecm) {
@@ -431,18 +474,32 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
final Object v = request.getParam(0);
// wrap transferred cache block into cacheable data
Data data;
- if(v instanceof CacheBlock)
- data = ExecutionContext.createCacheableData((CacheBlock) v);
- else if(v instanceof ScalarObject)
+ long size = 0;
+ if(v instanceof CacheBlock) {
+ var block = ExecutionContext.createCacheableData((CacheBlock) v);
+ size = block.getDataSize();
+ data = block;
+ }
+ else if(v instanceof ScalarObject) {
data = (ScalarObject) v;
- else if(v instanceof ListObject)
+ size = ((ScalarObject) v).getSize();
+ }
+ else if(v instanceof ListObject) {
data = (ListObject) v;
+ size = ((ListObject) v).getDataSize();
+ }
else if(request.getNumParams() == 2){
final Object v1= request.getParam(1);
- if(v1 == DataType.MATRIX)
- data = ExecutionContext.createMatrixObject((MatrixCharacteristics) v);
- else
- data = ExecutionContext.createFrameObject((MatrixCharacteristics) v);
+ if(v1 == DataType.MATRIX) {
+ var mtrx = ExecutionContext.createMatrixObject((MatrixCharacteristics) v);
+ size = mtrx.getDataSize();
+ data = mtrx;
+ }
+ else {
+ var frm = ExecutionContext.createFrameObject((MatrixCharacteristics) v);
+ size = frm.getDataSize();
+ data = frm;
+ }
}
else
throw new FederatedWorkerHandlerException(
@@ -452,6 +509,10 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
// set variable and construct empty response
ec.setVariable(varName, data);
+ if (DMLScript.STATISTICS){
+ FederatedStatistics.addDataObject(new DataObjectModel(varName, data.getDataType().name(), data.getValueType().name(), size));
+ }
+
if(shouldTryAsyncCompress())
CompressedMatrixBlockFactory.compressAsync(ec, varName);
@@ -500,8 +561,11 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
}
}
- private FederatedResponse execInstruction(FederatedRequest request, ExecutionContextMap ecm) throws Exception {
+ private FederatedResponse execInstruction(FederatedRequest request, ExecutionContextMap ecm, EventStageModel eventStage) throws Exception {
final Instruction ins = InstructionParser.parseSingleInstruction((String) request.getParam(0));
+
+ eventStage.operation = ins.getExtendedOpcode();
+
final long tid = request.getTID();
final ExecutionContext ec = getContextForInstruction(tid, ins, ecm);
setThreads(ins);
@@ -600,6 +664,7 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
private FederatedResponse execClear(ExecutionContextMap ecm) {
try {
ecm.clear();
+ FederatedStatistics.removeDataObjects();
}
catch(DMLPrivacyException | FederatedWorkerHandlerException ex) {
throw ex;
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Backend-architecture.svg b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Backend-architecture.svg
new file mode 100644
index 0000000000..fe3127736d
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Backend-architecture.svg
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Do not edit this file with editors other than diagrams.net -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" style="background-color: rgb(255, 255, 255);" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="691px" height="248px" viewBox="-0.5 -0.5 691 248" content="<mxfile host="app.diagrams.net" modified="2022-06-25T15:19:14.599Z" agent="5.0 (X11)" etag="V-eVfgA_e9BLHzG7TU7e" version="20.0.3" type="google"><diagram name="Page-1" id="2a216829-ef6e-dabb-86c1-c78 [...]
\ No newline at end of file
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Backend-processes.svg b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Backend-processes.svg
new file mode 100644
index 0000000000..7e4d8d7dd8
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Backend-processes.svg
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Do not edit this file with editors other than diagrams.net -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" style="background-color: rgb(255, 255, 255);" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="799px" height="651px" viewBox="-0.5 -0.5 799 651" content="<mxfile host="app.diagrams.net" modified="2022-06-24T15:30:45.189Z" agent="5.0 (X11)" etag="RVybFdfQ_E3N6H4QiMxs" version="20.0.3" type="google"><diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1"& [...]
\ No newline at end of file
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/DB-diagram.svg b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/DB-diagram.svg
new file mode 100644
index 0000000000..8fd7a100a8
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/DB-diagram.svg
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Do not edit this file with editors other than diagrams.net -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" style="background-color: rgb(255, 255, 255);" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="361px" height="254px" viewBox="-0.5 -0.5 361 254" content="<mxfile host="app.diagrams.net" modified="2022-06-24T15:02:11.930Z" agent="5.0 (X11)" etag="xU0YAODbRBLnMw9GYvGD" version="19.0.3" type="google"><diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1"& [...]
\ No newline at end of file
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java
index 8976d65194..031866c279 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServer.java
@@ -23,11 +23,16 @@ import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpServerCodec;
+import io.netty.handler.codec.http.cors.CorsConfig;
+import io.netty.handler.codec.http.cors.CorsConfigBuilder;
+import io.netty.handler.codec.http.cors.CorsHandler;
import org.apache.log4j.Logger;
public class FederatedMonitoringServer {
@@ -50,6 +55,16 @@ public class FederatedMonitoringServer {
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
+ var corsConfig = CorsConfigBuilder.forAnyOrigin()
+ .allowedRequestHeaders("*")
+ .allowedRequestMethods(
+ HttpMethod.DELETE,
+ HttpMethod.GET,
+ HttpMethod.PUT,
+ HttpMethod.POST,
+ HttpMethod.OPTIONS)
+ .build();
+
ServerBootstrap server = new ServerBootstrap();
server.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
@@ -59,10 +74,13 @@ public class FederatedMonitoringServer {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast(new HttpServerCodec());
+ pipeline.addLast(new CorsHandler(corsConfig));
pipeline.addLast(new FederatedMonitoringServerHandler());
}
});
+ server.childOption(ChannelOption.SO_KEEPALIVE, true);
+
log.info("Starting Federated Monitoring Backend server at port: " + _port);
ChannelFuture f = server.bind(_port).sync();
log.info("Started Federated Monitoring Backend at port: " + _port);
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServerHandler.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServerHandler.java
index 2e7006055b..3bdcab8f4d 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServerHandler.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/FederatedMonitoringServerHandler.java
@@ -23,14 +23,15 @@ import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpContent;
import io.netty.handler.codec.http.HttpObject;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.LastHttpContent;
import io.netty.util.CharsetUtil;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers.IController;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers.CoordinatorController;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers.StatisticsController;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers.WorkerController;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Request;
import java.util.HashMap;
import java.util.Map;
@@ -44,31 +45,40 @@ public class FederatedMonitoringServerHandler extends SimpleChannelInboundHandle
{
_allControllers.put("/coordinators", new CoordinatorController());
_allControllers.put("/workers", new WorkerController());
+ _allControllers.put("/statistics", new StatisticsController());
}
- private final static ThreadLocal<Request> _currentRequest = new ThreadLocal<>();
+ private static Request currentRequest = new Request();
+ private static final StringBuilder requestData = new StringBuilder();
@Override
protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) {
- if (msg instanceof LastHttpContent) {
- ByteBuf jsonBuf = ((LastHttpContent) msg).content();
- Request request = _currentRequest.get();
- request.setBody(jsonBuf.toString(CharsetUtil.UTF_8));
-
- _currentRequest.remove();
-
- final FullHttpResponse response = processRequest(request);
- ctx.write(response);
-
- } else if (msg instanceof HttpRequest) {
+ if (msg instanceof HttpRequest) {
HttpRequest httpRequest = (HttpRequest) msg;
Request request = new Request();
request.setContext(httpRequest);
- _currentRequest.set(request);
+ currentRequest = request;
}
+ if (msg instanceof HttpContent) {
+ ByteBuf jsonBuf = ((HttpContent) msg).content();
+ requestData.append(jsonBuf.toString(CharsetUtil.UTF_8));
+
+ if (msg instanceof LastHttpContent) {
+ Request request = currentRequest;
+
+ if (request != null && request.getContext() != null) {
+ request.setBody(requestData.toString());
+ final FullHttpResponse response = processRequest(request);
+ ctx.write(response);
+ }
+
+ requestData.setLength(0);
+ currentRequest = null;
+ }
+ }
}
@Override
@@ -83,30 +93,26 @@ public class FederatedMonitoringServerHandler extends SimpleChannelInboundHandle
}
private FullHttpResponse processRequest(final Request request) {
- try {
- final IController controller = parseController(request.getContext().uri());
- final String method = request.getContext().method().name();
+ final IController controller = parseController(request.getContext().uri());
+ final String method = request.getContext().method().name();
- switch (method) {
- case "GET":
- final Long id = parseId(request.getContext().uri());
+ switch (method) {
+ case "GET":
+ final Long id = parseId(request.getContext().uri());
- if (id != null) {
- return controller.get(request, id);
- }
+ if (id != null) {
+ return controller.get(request, id);
+ }
- return controller.getAll(request);
- case "PUT":
+ return controller.getAll(request);
+ case "PUT":
return controller.update(request, parseId(request.getContext().uri()));
- case "POST":
- return controller.create(request);
- case "DELETE":
- return controller.delete(request, parseId(request.getContext().uri()));
- default:
- throw new IllegalArgumentException("Method is not supported!");
- }
- } catch (RuntimeException ex) {
- throw ex;
+ case "POST":
+ return controller.create(request);
+ case "DELETE":
+ return controller.delete(request, parseId(request.getContext().uri()));
+ default:
+ throw new IllegalArgumentException("Method is not supported!");
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/README.md b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/README.md
new file mode 100644
index 0000000000..1a82b1d804
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/README.md
@@ -0,0 +1,71 @@
+
+# Backend for monitoring tool of federated infrastructure
+
+A backend application, used to collect, store, aggregate and return metrics data from coordinators and workers in the cluster
+
+
+## Install & Run
+
+The backend process can be started in a similar manner with how a worker is started:
+
+```bash
+ cd systemds
+ mvn package
+ ./bin/systemds [-r] FEDMONITOR [SystemDS.jar] <portnumber> [arguments]
+```
+
+Or with the specified **-fedMonitor 8080** flag indicating the start of the backend process on the specified port, in our case **8080**.
+
+## Main components
+
+### Architecture
+The following diagram illustrates the processes running in the backend.
+
+
+
+
+#### Controller
+Serves as the main integration point between the frontend and backend.
+
+#### Service
+Holds the business logic of the backend application.
+
+#### Repository
+serves as the main integration point between the backend and the chosen persistent storage. It can be extended to persist data in the file system, by extending the **IRepository** class and changing the instance in the service classes.
+
+### Database schema
+The following diagram illustrates the current state of the database schema.
+
+
+
+
+**Important to note**
+- There is no foreign key constraint between the worker and statistics tables.
+- The field for **coordinatorTraffic** is parsed into JSON format upon retrieval and saved as a string in the database. Example:
+```json
+{
+ "datetime": "2022-06-24T17:08:56.897188",
+ "coordinatorAddress": "localhost:8445",
+ "byteAmount": 45000
+}
+```
+- The field for **heavyHitters** is parsed into JSON format upon retrieval and saved as a string in the database. Example:
+```json
+{
+ "instruction": "fed_uamin",
+ "count": 4,
+ "duration": 0.5
+}
+```
+
+### Processes
+The following diagram illustrates the processes running in the backend.
+
+
+
+
+#### Statistics collection thread
+There is a dedicated thread for the communication between the backend and the workers and statistics are gathered periodically (every 3 seconds by default).
+
+#### Request processing
+The main logic of the application listens for REST requests coming from the frontend.
\ No newline at end of file
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Request.java
similarity index 98%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
copy to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Request.java
index 21d6812644..f5ea418b82 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Request.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
+package org.apache.sysds.runtime.controlprogram.federated.monitoring;
import io.netty.handler.codec.http.HttpRequest;
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Response.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Response.java
similarity index 82%
rename from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Response.java
rename to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Response.java
index 9693af6060..733fe0f17b 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Response.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/Response.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
+package org.apache.sysds.runtime.controlprogram.federated.monitoring;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.http.DefaultFullHttpResponse;
@@ -45,6 +45,18 @@ public class Response {
HttpResponseStatus.NOT_FOUND,
Unpooled.wrappedBuffer(exception.getBytes()));
+ response.headers().set(HttpHeaderNames.CONTENT_TYPE, "application/json");
+ response.headers().set(HttpHeaderNames.CONTENT_LENGTH, response.content().readableBytes());
+
+ return response;
+ }
+
+ public static FullHttpResponse forbidden(final String exception) {
+ FullHttpResponse response = new DefaultFullHttpResponse(
+ HttpVersion.HTTP_1_1,
+ HttpResponseStatus.FORBIDDEN,
+ Unpooled.wrappedBuffer(exception.getBytes()));
+
response.headers().set(HttpHeaderNames.CONTENT_TYPE, "text/plain");
response.headers().set(HttpHeaderNames.CONTENT_LENGTH, response.content().readableBytes());
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/Constants.java
similarity index 76%
rename from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
rename to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/Constants.java
index 18b17ea7fc..0fc066b6a3 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/EntityEnum.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/Constants.java
@@ -17,10 +17,10 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
+package org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers;
-public enum EntityEnum {
- WORKER,
- WORKER_STATS,
- COORDINATOR
+public class Constants {
+ public static final String GENERIC_SUCCESS_MSG = "{\"message\": \"Success\"}";
+ public static final String NOT_FOUND_MSG = "{\"message\": \"Entity not found\"}";
+ public static final String ID_JSON_MSG = "{\"id\": %d}";
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/CoordinatorController.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/CoordinatorController.java
index c6e4041542..fde6f1b713 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/CoordinatorController.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/CoordinatorController.java
@@ -20,46 +20,49 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers;
import io.netty.handler.codec.http.FullHttpResponse;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Request;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Response;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.CoordinatorModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.Request;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.Response;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.CoordinatorService;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.MapperService;
public class CoordinatorController implements IController {
- private final CoordinatorService _coordinatorService = new CoordinatorService();
+ private final CoordinatorService coordinatorService = new CoordinatorService();
@Override
public FullHttpResponse create(Request request) {
- var model = MapperService.getModelFromBody(request);
+ var model = MapperService.getModelFromBody(request, CoordinatorModel.class);
+ model.generateMonitoringKey();
- _coordinatorService.create(model);
+ model.id = coordinatorService.create(model);
- return Response.ok("Success");
+ return Response.ok(model.toString());
}
@Override
public FullHttpResponse update(Request request, Long objectId) {
- var model = MapperService.getModelFromBody(request);
+ var model = MapperService.getModelFromBody(request, CoordinatorModel.class);
+ model.generateMonitoringKey();
- _coordinatorService.update(model);
+ coordinatorService.update(model);
- return Response.ok("Success");
+ return Response.ok(model.toString());
}
@Override
public FullHttpResponse delete(Request request, Long objectId) {
- _coordinatorService.remove(objectId);
+ coordinatorService.remove(objectId);
- return Response.ok("Success");
+ return Response.ok(Constants.GENERIC_SUCCESS_MSG);
}
@Override
public FullHttpResponse get(Request request, Long objectId) {
- var result = _coordinatorService.get(objectId);
+ var result = coordinatorService.get(objectId);
if (result == null) {
- return Response.notFound("No such coordinator can be found");
+ return Response.notFound(Constants.NOT_FOUND_MSG);
}
return Response.ok(result.toString());
@@ -67,11 +70,7 @@ public class CoordinatorController implements IController {
@Override
public FullHttpResponse getAll(Request request) {
- var coordinators = _coordinatorService.getAll();
-
- if (coordinators.isEmpty()) {
- return Response.notFound("No coordinators can be found");
- }
+ var coordinators = coordinatorService.getAll();
return Response.ok(coordinators.toString());
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java
index 6016748bc8..17a6df58be 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java
@@ -20,7 +20,7 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers;
import io.netty.handler.codec.http.FullHttpResponse;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Request;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.Request;
public interface IController {
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/StatisticsController.java
similarity index 65%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java
copy to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/StatisticsController.java
index 63f68a6e86..5fdeba1b09 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/StatisticsController.java
@@ -20,60 +20,42 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers;
import io.netty.handler.codec.http.FullHttpResponse;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Request;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Response;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.MapperService;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.WorkerService;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.Request;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.Response;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatisticsOptions;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.StatisticsService;
-public class WorkerController implements IController {
-
- private final WorkerService _workerService = new WorkerService();
+public class StatisticsController implements IController {
+ private final StatisticsService statisticsService = new StatisticsService();
@Override
public FullHttpResponse create(Request request) {
-
- var model = MapperService.getModelFromBody(request);
-
- _workerService.create(model);
-
- return Response.ok("Success");
+ return Response.forbidden("");
}
@Override
public FullHttpResponse update(Request request, Long objectId) {
- var model = MapperService.getModelFromBody(request);
-
- _workerService.update(model);
-
- return Response.ok("Success");
+ return Response.forbidden("");
}
@Override
public FullHttpResponse delete(Request request, Long objectId) {
- _workerService.remove(objectId);
-
- return Response.ok("Success");
+ return Response.forbidden("");
}
@Override
public FullHttpResponse get(Request request, Long objectId) {
- var result = _workerService.get(objectId);
- if (result == null) {
- return Response.notFound("No such worker can be found");
- }
+ // Creates options with the default values
+ var options = new StatisticsOptions();
+
+ var result = statisticsService.getAll(objectId, options);
return Response.ok(result.toString());
}
@Override
public FullHttpResponse getAll(Request request) {
- var workers = _workerService.getAll();
-
- if (workers.isEmpty()) {
- return Response.notFound("No workers can be found");
- }
-
- return Response.ok(workers.toString());
+ return Response.forbidden("");
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java
index 63f68a6e86..52e318182e 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/WorkerController.java
@@ -20,58 +20,62 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers;
import io.netty.handler.codec.http.FullHttpResponse;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Request;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Response;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.Request;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.Response;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.MapperService;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.StatisticsService;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.WorkerService;
public class WorkerController implements IController {
-
- private final WorkerService _workerService = new WorkerService();
+ private final WorkerService workerService = new WorkerService();
@Override
public FullHttpResponse create(Request request) {
- var model = MapperService.getModelFromBody(request);
+ var model = MapperService.getModelFromBody(request, WorkerModel.class);
- _workerService.create(model);
+ model.id = workerService.create(model);
- return Response.ok("Success");
+ return Response.ok(model.toString());
}
@Override
public FullHttpResponse update(Request request, Long objectId) {
- var model = MapperService.getModelFromBody(request);
+ var model = MapperService.getModelFromBody(request, WorkerModel.class);
- _workerService.update(model);
+ workerService.update(model);
+ model.setOnlineStatus(workerService.getWorkerOnlineStatus(model.id));
- return Response.ok("Success");
+ return Response.ok(model.toString());
}
@Override
public FullHttpResponse delete(Request request, Long objectId) {
- _workerService.remove(objectId);
+ workerService.remove(objectId);
- return Response.ok("Success");
+ return Response.ok(Constants.GENERIC_SUCCESS_MSG);
}
@Override
public FullHttpResponse get(Request request, Long objectId) {
- var result = _workerService.get(objectId);
+ var result = workerService.get(objectId);
if (result == null) {
- return Response.notFound("No such worker can be found");
+ return Response.notFound(Constants.NOT_FOUND_MSG);
}
+ result.setOnlineStatus(workerService.getWorkerOnlineStatus(result.id));
+
return Response.ok(result.toString());
}
@Override
public FullHttpResponse getAll(Request request) {
- var workers = _workerService.getAll();
+ var workers = workerService.getAll();
- if (workers.isEmpty()) {
- return Response.notFound("No workers can be found");
+ for (var worker: workers) {
+ worker.setOnlineStatus(workerService.getWorkerOnlineStatus(worker.id));
}
return Response.ok(workers.toString());
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseModel.java
similarity index 89%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
copy to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseModel.java
index 41cf507696..dbd3320c01 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseModel.java
@@ -19,4 +19,8 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-public abstract class BaseEntityModel { }
+import java.io.Serializable;
+
+public abstract class BaseModel implements Serializable {
+ public Long id;
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java
similarity index 52%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
copy to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java
index 21d6812644..5999f59a11 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java
@@ -19,25 +19,30 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import io.netty.handler.codec.http.HttpRequest;
+import java.io.Serializable;
+import java.time.LocalDateTime;
-public class Request {
- private HttpRequest _context;
- private String _body;
+public abstract class CoordinatorConnectionModel extends BaseModel {
+ public Long coordinatorId;
+ private String coordinatorHostId;
+ private static final String localhostIp = "127.0.0.1";
+ private static final String localhostString = "localhost";
- public HttpRequest getContext() {
- return _context;
- }
+ public CoordinatorConnectionModel() { }
- public void setContext(final HttpRequest requestContext) {
- this._context = requestContext;
+ public void setCoordinatorHostId(String hostId) {
+ this.coordinatorHostId = hostId;
}
- public String getBody() {
- return _body;
- }
+ public String getCoordinatorHostId() {
+ this.coordinatorHostId = this.coordinatorHostId.replaceFirst("/", "");
+
+ if (this.coordinatorHostId.contains(localhostIp)) {
+ this.coordinatorHostId = this.coordinatorHostId.replace(localhostIp, localhostString);
+ }
+
+ this.coordinatorHostId = this.coordinatorHostId.replaceFirst(":\\d+", "");
- public void setBody(final String content) {
- this._body = content;
+ return this.coordinatorHostId;
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorModel.java
similarity index 56%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
copy to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorModel.java
index 21d6812644..98f25ee267 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorModel.java
@@ -19,25 +19,35 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import io.netty.handler.codec.http.HttpRequest;
+public class CoordinatorModel extends BaseModel {
+ public String name;
+ public String host;
+ public Long processId;
+ public String monitoringHostIdKey;
-public class Request {
- private HttpRequest _context;
- private String _body;
+ private static final String keyFormat = "%s-%d";
- public HttpRequest getContext() {
- return _context;
+ private static final String JsonFormat = "{" +
+ "\"id\": %d," +
+ "\"name\": \"%s\"," +
+ "\"host\": \"%s\"," +
+ "\"processId\": %d" +
+ "}";
+
+ public CoordinatorModel(final Long id) {
+ this.id = id;
}
- public void setContext(final HttpRequest requestContext) {
- this._context = requestContext;
+ public CoordinatorModel() {
+ this(-1L);
}
- public String getBody() {
- return _body;
+ public void generateMonitoringKey() {
+ this.monitoringHostIdKey = String.format(keyFormat, host, processId);
}
- public void setBody(final String content) {
- this._body = content;
+ @Override
+ public String toString() {
+ return String.format(JsonFormat, super.id, this.name, this.host, this.processId);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java
new file mode 100644
index 0000000000..b8c218c749
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+public class DataObjectModel extends BaseModel {
+
+ public Long workerId;
+ public String varName;
+ public String dataType;
+ public String valueType;
+ public Long size;
+
+ private static final String JsonFormat = "{" +
+ "\"varName\": \"%s\"," +
+ "\"dataType\": \"%s\"," +
+ "\"valueType\": \"%s\"," +
+ "\"size\": %d" +
+ "}";
+
+ public DataObjectModel() {
+ this(-1L);
+ }
+
+ private DataObjectModel(final Long id) {
+ this.id = id;
+ }
+
+ public DataObjectModel(final String varName, final String dataType, final String valueType, final Long size) {
+ this(-1L, varName, dataType, valueType, size);
+ }
+
+ public DataObjectModel(final Long id, final String varName, final String dataType, final String valueType, final Long size) {
+ this.id = id;
+ this.varName = varName;
+ this.dataType = dataType;
+ this.valueType = valueType;
+ this.size = size;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(JsonFormat, this.varName, this.dataType, this.valueType, this.size);
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java
new file mode 100644
index 0000000000..b8e3bd85cb
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class EventModel extends CoordinatorConnectionModel {
+
+ public Long workerId;
+ private String coordinatorName;
+ public List<EventStageModel> stages;
+
+ private static final String JsonFormat = "{" +
+ "\"coordinatorName\": \"%s\"," +
+ "\"stages\": [%s]" +
+ "}";
+
+ public EventModel() {
+ this(-1L);
+ }
+
+ private EventModel(final Long id) {
+ this.id = id;
+ this.stages = new ArrayList<>();
+ }
+
+ public EventModel(final Long workerId, final Long coordinatorId) {
+ this(-1L, workerId, coordinatorId);
+ }
+
+ public EventModel(final Long id, final Long workerId, final Long coordinatorId) {
+ this.id = id;
+ this.workerId = workerId;
+ this.coordinatorId = coordinatorId;
+ this.stages = new ArrayList<>();
+ }
+
+ public void setCoordinatorName(String name) {
+ this.coordinatorName = name;
+ }
+
+ @Override
+ public String toString() {
+ String stagesStr = this.stages.stream()
+ .map(EventStageModel::toString)
+ .collect(Collectors.joining(","));
+
+ return String.format(JsonFormat, this.coordinatorName, stagesStr);
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java
similarity index 51%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
copy to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java
index 21d6812644..cd26bfbdb4 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java
@@ -19,25 +19,42 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import io.netty.handler.codec.http.HttpRequest;
+import java.io.Serializable;
+import java.time.LocalDateTime;
-public class Request {
- private HttpRequest _context;
- private String _body;
+public class EventStageModel extends BaseModel {
- public HttpRequest getContext() {
- return _context;
+ public Long eventId;
+ public String operation;
+ public LocalDateTime startTime;
+ public LocalDateTime endTime;
+
+ private static final String JsonFormat = "{" +
+ "\"operation\": \"%s\"," +
+ "\"startTime\": \"%s\"," +
+ "\"endTime\": \"%s\"" +
+ "}";
+
+ public EventStageModel() {
+ this(-1L);
+ }
+
+ private EventStageModel(final Long id) {
+ this.id = id;
}
- public void setContext(final HttpRequest requestContext) {
- this._context = requestContext;
+ public EventStageModel(final Long eventId, final String stageOperation) {
+ this(-1L, eventId, stageOperation);
}
- public String getBody() {
- return _body;
+ public EventStageModel(final Long id, final Long eventId, final String operation) {
+ this.id = id;
+ this.eventId = eventId;
+ this.operation = operation;
}
- public void setBody(final String content) {
- this._body = content;
+ @Override
+ public String toString() {
+ return String.format(JsonFormat, this.operation, this.startTime, this.endTime);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/NodeEntityModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/NodeEntityModel.java
deleted file mode 100644
index 725274509f..0000000000
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/NodeEntityModel.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-
-import java.util.List;
-
-public class NodeEntityModel extends BaseEntityModel {
- private Long _id;
- private String _name;
- private String _address;
-
- private List<BaseEntityModel> _stats;
-
- public NodeEntityModel() { }
-
- public NodeEntityModel(final Long id, final String name, final String address) {
- _id = id;
- _name = name;
- _address = address;
- }
-
- public Long getId() {
- return _id;
- }
-
- public void setId(final Long id) {
- _id = id;
- }
-
- public String getName() {
- return _name;
- }
-
- public void setName(final String name) {
- _name = name;
- }
-
- public String getAddress() {
- return _address;
- }
-
- public void setAddress(final String address) {
- _address = address;
- }
-
- public List<BaseEntityModel> getStats() {
- return _stats;
- }
-
- public void setStats(final List<BaseEntityModel> stats) {
- _stats = stats;
- }
-
- @Override
- public String toString() {
- return String.format("{" +
- "\"id\": %d," +
- "\"name\": \"%s\"," +
- "\"address\": \"%s\"," +
- "\"stats\": %s" +
- "}", _id, _name, _address, _stats);
- }
-}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/RequestModel.java
similarity index 58%
rename from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
rename to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/RequestModel.java
index 21d6812644..81171e9311 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/Request.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/RequestModel.java
@@ -19,25 +19,37 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import io.netty.handler.codec.http.HttpRequest;
+public class RequestModel extends CoordinatorConnectionModel {
-public class Request {
- private HttpRequest _context;
- private String _body;
+ public Long workerId;
+ public String type;
+ public Long count;
- public HttpRequest getContext() {
- return _context;
+ private static final String JsonFormat = "{" +
+ "\"type\": \"%s\"," +
+ "\"count\": %d" +
+ "}";
+
+ public RequestModel() {
+ this(-1L);
+ }
+
+ private RequestModel(final Long id) {
+ this.id = id;
}
- public void setContext(final HttpRequest requestContext) {
- this._context = requestContext;
+ public RequestModel(final String type, final Long count) {
+ this(-1L, type, count);
}
- public String getBody() {
- return _body;
+ public RequestModel(final Long id, final String type, final Long count) {
+ this.id = id;
+ this.type = type;
+ this.count = count;
}
- public void setBody(final String content) {
- this._body = content;
+ @Override
+ public String toString() {
+ return String.format(JsonFormat, this.type, this.count);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsModel.java
new file mode 100644
index 0000000000..cdbe6e73c5
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsModel.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class StatisticsModel extends BaseModel {
+ public List<UtilizationModel> utilization;
+ public List<TrafficModel> traffic;
+ public List<EventModel> events;
+ public List<DataObjectModel> dataObjects;
+ public List<RequestModel> requests;
+
+ private static final String JsonFormat = "{" +
+ "\"utilization\": [%s]," +
+ "\"traffic\": [%s]," +
+ "\"events\": [%s]," +
+ "\"dataObjects\": [%s]," +
+ "\"requests\": [%s]" +
+ "}";
+
+ public StatisticsModel() { }
+
+ public StatisticsModel(List<UtilizationModel> utilization,
+ List<TrafficModel> traffic,
+ List<EventModel> events,
+ List<DataObjectModel> dataObjects,
+ List<RequestModel> requests) {
+ this.utilization = utilization;
+ this.traffic = traffic;
+ this.events = events;
+ this.dataObjects = dataObjects;
+ this.requests = requests;
+ }
+
+
+ @Override
+ public String toString() {
+ String utilizationStr = null, trafficStr = null, eventsStr = null, dataObjectsStr = null, requestsStr = null;
+
+ if (utilization != null) {
+ utilizationStr = utilization.stream()
+ .map(UtilizationModel::toString)
+ .collect(Collectors.joining(","));
+ }
+
+ if (traffic != null) {
+ trafficStr = traffic.stream()
+ .map(TrafficModel::toString)
+ .collect(Collectors.joining(","));
+ }
+
+ if (events != null) {
+ eventsStr = events.stream()
+ .map(EventModel::toString)
+ .collect(Collectors.joining(","));
+ }
+
+ if (dataObjects != null) {
+ dataObjectsStr = dataObjects.stream()
+ .map(DataObjectModel::toString)
+ .collect(Collectors.joining(","));
+ }
+
+ if (requests != null) {
+ requestsStr = requests.stream()
+ .map(RequestModel::toString)
+ .collect(Collectors.joining(","));
+ }
+
+ return String.format(JsonFormat, utilizationStr, trafficStr, eventsStr, dataObjectsStr, requestsStr);
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsOptions.java
similarity index 78%
rename from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
rename to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsOptions.java
index 41cf507696..bcbc57357d 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseEntityModel.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsOptions.java
@@ -19,4 +19,11 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-public abstract class BaseEntityModel { }
+public class StatisticsOptions extends BaseModel {
+ public int rowCount = 20;
+ public boolean utilization = true;
+ public boolean traffic = true;
+ public boolean events = true;
+ public boolean dataObjects = true;
+ public boolean requests = true;
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatsEntityModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatsEntityModel.java
deleted file mode 100644
index bfd9c9e840..0000000000
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatsEntityModel.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.lang3.tuple.Triple;
-
-import java.time.LocalDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.List;
-import java.util.Map;
-
-public class StatsEntityModel extends BaseEntityModel {
- private Long _workerId;
- private double _cpuUsage;
- private double _memoryUsage;
- private Map<String, Pair<Long, Double>> _heavyHitterInstructionsObj;
- private String _heavyHitterInstructions;
- private List<Triple<LocalDateTime, String, Long>> _transferredBytesObj;
- private String _transferredBytes;
-
- public StatsEntityModel() { }
-
- public StatsEntityModel(Long workerId, double cpuUsage, double memoryUsage,
- Map<String, Pair<Long, Double>> heavyHitterInstructionsObj,
- List<Triple<LocalDateTime, String, Long>> transferredBytesObj)
- {
- _workerId = workerId;
- _cpuUsage = cpuUsage;
- _memoryUsage = memoryUsage;
- _heavyHitterInstructionsObj = heavyHitterInstructionsObj;
- _transferredBytesObj = transferredBytesObj;
- _heavyHitterInstructions = "";
- _transferredBytes = "";
- }
-
- public Long getWorkerId() {
- return _workerId;
- }
-
- public void setWorkerId(final Long workerId) {
- _workerId = workerId;
- }
-
- public double getCPUUsage() {
- return _cpuUsage;
- }
-
- public void setCPUUsage(final double cpuUsage) {
- _cpuUsage = cpuUsage;
- }
-
- public double getMemoryUsage() {
- return _memoryUsage;
- }
-
- public void setMemoryUsage(final double memoryUsage) {
- _memoryUsage = memoryUsage;
- }
-
- public String getHeavyHitterInstructions() {
- if (_heavyHitterInstructions.isEmpty() || _heavyHitterInstructions.isBlank()) {
- StringBuilder sb = new StringBuilder();
-
- sb.append("{");
- for(Map.Entry<String, Pair<Long, Double>> entry : _heavyHitterInstructionsObj.entrySet()) {
- String instruction = entry.getKey();
- Long count = entry.getValue().getLeft();
- double duration = entry.getValue().getRight();
- sb.append(String.format("{" +
- "\"instruction\": %s," +
- "\"count\": \"%d\"," +
- "\"duration\": \"%.2f\"," +
- "},", instruction, count, duration));
- }
- sb.append("}");
-
- _heavyHitterInstructions = sb.toString();
- }
-
- return _heavyHitterInstructions;
- }
-
- public void setHeavyHitterInstructions(final String heavyHitterInstructionsJsonString) {
- _heavyHitterInstructions = heavyHitterInstructionsJsonString;
- }
-
- public String getTransferredBytes() {
- if (_transferredBytes.isEmpty() || _transferredBytes.isBlank()) {
- StringBuilder sb = new StringBuilder();
-
- sb.append("{");
- for (var entry: _transferredBytesObj) {
- sb.append(String.format("{" +
- "\"datetime\": %s," +
- "\"coordinatorAddress\": \"%s\"," +
- "\"byteAmount\": \"%d\"," +
- "},", entry.getLeft().format(DateTimeFormatter.ISO_DATE_TIME),
- entry.getMiddle(), entry.getRight()));
- }
- sb.append("}");
-
- _transferredBytes = sb.toString();
- }
-
- return _transferredBytes;
- }
-
- public void setTransferredBytes(final String transferredBytesJsonString) {
- _transferredBytes = transferredBytesJsonString;
- }
-
- @Override
- public String toString() {
- return String.format("{" +
- "\"cpuUsage\": %.2f," +
- "\"memoryUsage\": %.2f," +
- "\"coordinatorTraffic\": %s," +
- "\"heavyHitters\": %s" +
- "}", _cpuUsage, _memoryUsage, getTransferredBytes(), getHeavyHitterInstructions());
- }
-}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java
new file mode 100644
index 0000000000..d4b4b860ea
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+public class TrafficModel extends CoordinatorConnectionModel {
+
+ public Long workerId;
+ public LocalDateTime timestamp;
+ public Long byteAmount;
+
+ private static final String JsonFormat = "{" +
+ "\"timestamp\": \"%s\"," +
+ "\"coordinatorId\": %d," +
+ "\"byteAmount\": %d" +
+ "}";
+
+ public TrafficModel() { }
+
+ public TrafficModel(final LocalDateTime timestamp, final String coordinatorHostId, final Long byteAmount) {
+ this.timestamp = timestamp;
+ this.byteAmount = byteAmount;
+
+ super.setCoordinatorHostId(coordinatorHostId);
+ }
+
+ private TrafficModel(final Long id) {
+ this.id = id;
+ }
+
+ public TrafficModel(final Long workerId, final String coordinatorAddress, final Long byteAmount) {
+ this(-1L, workerId, LocalDateTime.now(), coordinatorAddress, byteAmount);
+ }
+
+ public TrafficModel(final Long id, final Long workerId, final LocalDateTime timestamp, final String coordinatorHostId, final Long byteAmount) {
+ this.id = id;
+ this.workerId = workerId;
+ this.timestamp = timestamp;
+ this.byteAmount = byteAmount;
+
+ super.setCoordinatorHostId(coordinatorHostId);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(JsonFormat, this.timestamp, this.coordinatorId, this.byteAmount);
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java
new file mode 100644
index 0000000000..83a5d290e4
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+public class UtilizationModel extends BaseModel {
+
+ public Long workerId;
+ public LocalDateTime timestamp;
+ public double cpuUsage;
+ public double memoryUsage;
+
+ private static final String JsonFormat = "{" +
+ "\"timestamp\": \"%s\"," +
+ "\"cpuUsage\": %.2f," +
+ "\"memoryUsage\": %.2f" +
+ "}";
+
+ public UtilizationModel() {
+ this(-1L);
+ }
+
+ private UtilizationModel(final Long id) {
+ this.id = id;
+ }
+
+ public UtilizationModel(final double cpuUsage, final double memoryUsage) {
+ this(-1L, -1L, LocalDateTime.now(), cpuUsage, memoryUsage);
+ }
+
+ public UtilizationModel(final Long workerId, final double cpuUsage, final double memoryUsage) {
+ this(-1L, workerId, LocalDateTime.now(), cpuUsage, memoryUsage);
+ }
+
+ public UtilizationModel(final Long id, final Long workerId, final LocalDateTime timestamp, final double cpuUsage, final double memoryUsage) {
+ this.id = id;
+ this.workerId = workerId;
+ this.timestamp = timestamp;
+ this.cpuUsage = cpuUsage;
+ this.memoryUsage = memoryUsage;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(JsonFormat, this.timestamp, this.cpuUsage, this.memoryUsage);
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java
similarity index 56%
copy from src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java
copy to src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java
index 6016748bc8..bd26e5235d 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/controllers/IController.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java
@@ -17,20 +17,43 @@
* under the License.
*/
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.controllers;
+package org.apache.sysds.runtime.controlprogram.federated.monitoring.models;
-import io.netty.handler.codec.http.FullHttpResponse;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Request;
+import java.util.List;
-public interface IController {
+public class WorkerModel extends BaseModel {
+ public String name;
+ public String address;
- FullHttpResponse create(final Request request);
+ private boolean isOnline;
- FullHttpResponse update(final Request request, final Long objectId);
+ private static final String JsonFormat = "{" +
+ "\"id\": %d," +
+ "\"name\": \"%s\"," +
+ "\"address\": \"%s\"," +
+ "\"isOnline\": %b" +
+ "}";
- FullHttpResponse delete(final Request request, final Long objectId);
+ public WorkerModel(final Long id) {
+ this.id = id;
+ }
- FullHttpResponse get(final Request request, final Long objectId);
+ public void setOnlineStatus(boolean status) {
+ this.isOnline = status;
+ }
- FullHttpResponse getAll(final Request request);
+ public WorkerModel() {
+ this(-1L);
+ }
+
+ public WorkerModel(final Long id, final String name, final String address) {
+ this.id = id;
+ this.name = name;
+ this.address = address;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(JsonFormat, super.id, this.name, this.address, this.isOnline);
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/Constants.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/Constants.java
index 40ce052716..c158138f04 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/Constants.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/Constants.java
@@ -20,15 +20,15 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
public class Constants {
- public static final String WORKERS_TABLE_NAME= "workers";
- public static final String COORDINATORS_TABLE_NAME= "coordinators";
- public static final String STATS_TABLE_NAME= "statistics";
+ public static final String ENTITY_CLASS_SUFFIX = "Model";
+ public static final String ENTITY_STRING_COL = "VARCHAR(5000)";
+ public static final String ENTITY_DOUBLE_COL = "DOUBLE";
+ public static final String ENTITY_NUMBER_COL = "INTEGER";
+ public static final String ENTITY_TIMESTAMP_COL = "TIMESTAMP";
public static final String ENTITY_NAME_COL = "name";
- public static final String ENTITY_ADDR_COL = "address";
- public static final String ENTITY_CPU_COL = "cpuUsage";
- public static final String ENTITY_MEM_COL = "memoryUsage";
- public static final String ENTITY_TRAFFIC_COL = "coordinatorTraffic";
- public static final String ENTITY_HEAVY_HITTERS_COL = "heavyHitters";
+ public static final String ENTITY_ADDRESS_COL = "address";
+ public static final String ENTITY_MONITORING_KEY_COL = "monitoringHostIdKey";
public static final String ENTITY_ID_COL = "id";
public static final String ENTITY_WORKER_ID_COL = "workerId";
+ public static final String ENTITY_EVENT_ID_COL = "eventId";
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/DerbyRepository.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/DerbyRepository.java
index 02a948769f..f19bde7b4f 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/DerbyRepository.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/DerbyRepository.java
@@ -19,39 +19,44 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.NodeEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatsEntityModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.*;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.MapperService;
+import java.lang.reflect.Field;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
public class DerbyRepository implements IRepository {
private final static String DB_CONNECTION = "jdbc:derby:memory:derbyDB";
private final Connection _db;
+ private final List<BaseModel> _allEntities = new ArrayList<>(List.of(
+ new WorkerModel(),
+ new CoordinatorModel(),
+ new UtilizationModel(),
+ new TrafficModel(),
+ new EventModel(),
+ new EventStageModel(),
+ new DataObjectModel(),
+ new RequestModel()
+ ));
private static final String ENTITY_SCHEMA_CREATE_STMT = "CREATE TABLE %s " +
- "(id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1), " +
- "%s VARCHAR(60), " +
- "%s VARCHAR(120))";
- private static final String ENTITY_SCHEMA_CREATE_STATS_STMT = "CREATE TABLE %s " +
- "(id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1), " +
- "%s INTEGER, " +
- "%s DOUBLE, " +
- "%s DOUBLE," +
- "%s VARCHAR(1000)," +
- "%s VARCHAR(1000))";
- private static final String ENTITY_INSERT_STMT = "INSERT INTO %s (%s, %s) VALUES (?, ?)";
- private static final String ENTITY_STATS_INSERT_STMT = "INSERT INTO %s (%s, %s, %s, %s, %s) VALUES (?, ?, ?, ?, ?)";
+ "(id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1)";
+ private static final String ENTITY_INSERT_STMT = "INSERT INTO %s VALUES %s";
private static final String GET_ENTITY_WITH_COL_STMT = "SELECT * FROM %s WHERE %s = ?";
+ private static final String GET_ENTITY_WITH_COL_LIMIT_STMT = "SELECT * FROM %s " +
+ "WHERE %s = ? " +
+ "ORDER BY ID DESC " +
+ "FETCH FIRST %d ROWS ONLY";
private static final String DELETE_ENTITY_WITH_COL_STMT = "DELETE FROM %s WHERE %s = ?";
- private static final String UPDATE_ENTITY_WITH_COL_STMT = "UPDATE %s SET %s = ?, %s = ? WHERE %s = ?";
+ private static final String UPDATE_ENTITY_WITH_COL_STMT = "UPDATE %s SET %s WHERE %s = ?";
private static final String GET_ALL_ENTITIES_STMT = "SELECT * FROM %s";
public DerbyRepository() {
@@ -75,33 +80,41 @@ public class DerbyRepository implements IRepository {
private void createMonitoringEntitiesInDB(Connection db) {
try {
var dbMetaData = db.getMetaData();
- var workersExist = dbMetaData.getTables(null, null, Constants.WORKERS_TABLE_NAME.toUpperCase(),null);
- var statsExist = dbMetaData.getTables(null, null, Constants.STATS_TABLE_NAME.toUpperCase(),null);
- var coordinatorsExist = dbMetaData.getTables(null, null, Constants.COORDINATORS_TABLE_NAME.toUpperCase(),null);
-
- // Check if table already exists and create if not
- if(!workersExist.next()) {
- PreparedStatement st = db.prepareStatement(
- String.format(ENTITY_SCHEMA_CREATE_STMT, Constants.WORKERS_TABLE_NAME, Constants.ENTITY_NAME_COL, Constants.ENTITY_ADDR_COL));
- st.executeUpdate();
- }
- if(!statsExist.next()) {
- PreparedStatement st = db.prepareStatement(
- String.format(ENTITY_SCHEMA_CREATE_STATS_STMT, Constants.STATS_TABLE_NAME,
- Constants.ENTITY_WORKER_ID_COL,
- Constants.ENTITY_CPU_COL,
- Constants.ENTITY_MEM_COL,
- Constants.ENTITY_TRAFFIC_COL,
- Constants.ENTITY_HEAVY_HITTERS_COL));
- st.executeUpdate();
- }
+ for (var entity: _allEntities) {
+ var entityName = entity.getClass().getSimpleName().replace(Constants.ENTITY_CLASS_SUFFIX, "");
+ var entityExist = dbMetaData.getTables(null, null, entityName.toUpperCase(),null);
+
+ if(!entityExist.next()) {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(String.format(ENTITY_SCHEMA_CREATE_STMT, entityName));
+
+ var fields = entity.getClass().getFields();
+ for (var field: fields) {
- if(!coordinatorsExist.next()) {
- PreparedStatement st = db.prepareStatement(String.format(
- ENTITY_SCHEMA_CREATE_STMT, Constants.COORDINATORS_TABLE_NAME,
- Constants.ENTITY_NAME_COL, Constants.ENTITY_ADDR_COL));
- st.executeUpdate();
+ if (field.getName().equalsIgnoreCase(Constants.ENTITY_ID_COL)) {
+ continue;
+ }
+
+ if (field.getType().isAssignableFrom(String.class)) {
+ sb.append(String.format(",%s %s", field.getName(), Constants.ENTITY_STRING_COL));
+ } else if (field.getType().isAssignableFrom(double.class)) {
+ sb.append(String.format(",%s %s", field.getName(), Constants.ENTITY_DOUBLE_COL));
+ } else if (field.getType().isAssignableFrom(Long.class) ||
+ field.getType().isAssignableFrom(int.class)) {
+ sb.append(String.format(",%s %s", field.getName(), Constants.ENTITY_NUMBER_COL));
+ } else if (field.getType().isAssignableFrom(LocalDateTime.class)) {
+ sb.append(String.format(",%s %s", field.getName(), Constants.ENTITY_TIMESTAMP_COL));
+ }
+
+ }
+
+ sb.append(")");
+
+ PreparedStatement st = db.prepareStatement(sb.toString());
+ st.executeUpdate();
+ }
}
}
catch (SQLException e) {
@@ -109,42 +122,63 @@ public class DerbyRepository implements IRepository {
}
}
- public Long createEntity(EntityEnum type, BaseEntityModel model) {
+ public <T extends BaseModel> Long createEntity(T model) {
PreparedStatement st = null;
long id = -1L;
try {
- if (type == EntityEnum.WORKER_STATS) {
- st = _db.prepareStatement(
- String.format(ENTITY_STATS_INSERT_STMT, Constants.STATS_TABLE_NAME,
- Constants.ENTITY_WORKER_ID_COL,
- Constants.ENTITY_CPU_COL,
- Constants.ENTITY_MEM_COL,
- Constants.ENTITY_TRAFFIC_COL,
- Constants.ENTITY_HEAVY_HITTERS_COL), PreparedStatement.RETURN_GENERATED_KEYS);
-
- StatsEntityModel newModel = (StatsEntityModel) model;
-
- st.setLong(1, newModel.getWorkerId());
- st.setDouble(2, newModel.getCPUUsage());
- st.setDouble(3, newModel.getMemoryUsage());
- st.setString(4, newModel.getTransferredBytes());
- st.setString(5, newModel.getHeavyHitterInstructions());
- } else {
- st = _db.prepareStatement(
- String.format(ENTITY_INSERT_STMT, Constants.WORKERS_TABLE_NAME, Constants.ENTITY_NAME_COL, Constants.ENTITY_ADDR_COL),
- PreparedStatement.RETURN_GENERATED_KEYS);
- NodeEntityModel newModel = (NodeEntityModel) model;
-
- if (type == EntityEnum.COORDINATOR) {
- st = _db.prepareStatement(
- String.format(ENTITY_INSERT_STMT, Constants.COORDINATORS_TABLE_NAME, Constants.ENTITY_NAME_COL, Constants.ENTITY_ADDR_COL),
- PreparedStatement.RETURN_GENERATED_KEYS);
+
+ StringBuilder sb = new StringBuilder();
+
+ var entityName = model.getClass().getSimpleName().replace(Constants.ENTITY_CLASS_SUFFIX, "");
+
+ sb.append(String.format("%s (", entityName));
+
+ var fields = model.getClass().getFields();
+ int dbFieldCount = 0;
+ for (var field: fields) {
+
+ if (field.getName().equalsIgnoreCase(Constants.ENTITY_ID_COL)) {
+ continue;
+ }
+
+ if (field.getType().isAssignableFrom(String.class) ||
+ field.getType().isAssignableFrom(double.class) ||
+ field.getType().isAssignableFrom(Long.class) ||
+ field.getType().isAssignableFrom(int.class) ||
+ field.getType().isAssignableFrom(LocalDateTime.class)) {
+ sb.append(String.format("%s,", field.getName()));
+ dbFieldCount++;
+ }
+ }
+
+ sb.replace(sb.length() - 1, sb.length(), ")");
+ String bindVarsStr = String.format("(%s)", String.join(",", Collections.nCopies(dbFieldCount, "?")));
+
+ st = _db.prepareStatement(String.format(ENTITY_INSERT_STMT, sb, bindVarsStr), PreparedStatement.RETURN_GENERATED_KEYS);
+
+ int bindVarIndex = 1;
+ for (var field: fields) {
+
+ if (field.getName().equalsIgnoreCase(Constants.ENTITY_ID_COL)) {
+ continue;
}
- st.setString(1, newModel.getName());
- st.setString(2, newModel.getAddress());
+ if (field.getType().isAssignableFrom(String.class)) {
+ st.setString(bindVarIndex, String.valueOf(field.get(model)));
+ bindVarIndex++;
+ } else if (field.getType().isAssignableFrom(double.class)) {
+ st.setDouble(bindVarIndex, (double) field.get(model));
+ bindVarIndex++;
+ } else if (field.getType().isAssignableFrom(Long.class) ||
+ field.getType().isAssignableFrom(int.class)) {
+ st.setLong(bindVarIndex, (long) field.get(model));
+ bindVarIndex++;
+ } else if (field.getType().isAssignableFrom(LocalDateTime.class)) {
+ st.setTimestamp(bindVarIndex, Timestamp.valueOf((LocalDateTime) field.get(model)));
+ bindVarIndex++;
+ }
}
st.executeUpdate();
@@ -154,33 +188,27 @@ public class DerbyRepository implements IRepository {
id = rs.getLong(1); // this is the auto-generated id key
}
- } catch (SQLException e) {
+ } catch (SQLException | IllegalAccessException e) {
throw new RuntimeException(e);
}
return id;
}
- public BaseEntityModel getEntity(EntityEnum type, Long id) {
- BaseEntityModel resultModel = null;
+ public <T extends BaseModel> T getEntity(Long id, Class<T> type) {
+ T resultModel = null;
try {
- PreparedStatement st = _db.prepareStatement(
- String.format(GET_ENTITY_WITH_COL_STMT, Constants.WORKERS_TABLE_NAME, Constants.ENTITY_ID_COL));
+ var entityName = type.getSimpleName().replace(Constants.ENTITY_CLASS_SUFFIX, "");
- if (type == EntityEnum.COORDINATOR) {
- st = _db.prepareStatement(
- String.format(GET_ENTITY_WITH_COL_STMT, Constants.COORDINATORS_TABLE_NAME, Constants.ENTITY_ID_COL));
- } else if (type == EntityEnum.WORKER_STATS) {
- st = _db.prepareStatement(
- String.format(GET_ENTITY_WITH_COL_STMT, Constants.STATS_TABLE_NAME, Constants.ENTITY_WORKER_ID_COL));
- }
+ PreparedStatement st = _db.prepareStatement(
+ String.format(GET_ENTITY_WITH_COL_STMT, entityName, Constants.ENTITY_ID_COL));
st.setLong(1, id);
var resultSet = st.executeQuery();
if (resultSet.next()){
- resultModel = MapperService.mapEntityToModel(resultSet, type);
+ resultModel = MapperService.mapResultToModel(resultSet, type);
}
} catch (SQLException e) {
throw new RuntimeException(e);
@@ -189,21 +217,18 @@ public class DerbyRepository implements IRepository {
return resultModel;
}
- public List<BaseEntityModel> getAllEntities(EntityEnum type) {
- List<BaseEntityModel> resultModels = new ArrayList<>();
+ public <T extends BaseModel> List<T> getAllEntities(Class<T> type) {
+ List<T> resultModels = new ArrayList<>();
try {
- PreparedStatement st = _db.prepareStatement(
- String.format(GET_ALL_ENTITIES_STMT, Constants.WORKERS_TABLE_NAME));
+ var entityName = type.getSimpleName().replace(Constants.ENTITY_CLASS_SUFFIX, "");
- if (type == EntityEnum.COORDINATOR) {
- st = _db.prepareStatement(
- String.format(GET_ALL_ENTITIES_STMT, Constants.COORDINATORS_TABLE_NAME));
- }
+ PreparedStatement st = _db.prepareStatement(
+ String.format(GET_ALL_ENTITIES_STMT, entityName));
var resultSet = st.executeQuery();
while (resultSet.next()){
- resultModels.add(MapperService.mapEntityToModel(resultSet, type));
+ resultModels.add(MapperService.mapResultToModel(resultSet, type));
}
} catch (SQLException e) {
throw new RuntimeException(e);
@@ -211,23 +236,34 @@ public class DerbyRepository implements IRepository {
return resultModels;
}
+ public <T extends BaseModel> List<T> getAllEntitiesByField(String fieldName, Object value, Class<T> type) {
+ return getAllEntitiesByField(fieldName, value, type, -1);
+ }
- public List<BaseEntityModel> getAllEntitiesByField(EntityEnum type, Object fieldValue) {
- List<BaseEntityModel> resultModels = new ArrayList<>();
+ public <T extends BaseModel> List<T> getAllEntitiesByField(String fieldName, Object value, Class<T> type, int rowCount) {
+ List<T> resultModels = new ArrayList<>();
PreparedStatement st = null;
try {
- if (type == EntityEnum.WORKER_STATS) {
+ var entityName = type.getSimpleName().replace(Constants.ENTITY_CLASS_SUFFIX, "");
+
+ if (rowCount < 0) {
st = _db.prepareStatement(
- String.format(GET_ENTITY_WITH_COL_STMT, Constants.STATS_TABLE_NAME, Constants.ENTITY_WORKER_ID_COL));
- st.setLong(1, (Long) fieldValue);
+ String.format(GET_ENTITY_WITH_COL_STMT, entityName, fieldName));
} else {
- throw new NotImplementedException();
+ st = _db.prepareStatement(
+ String.format(GET_ENTITY_WITH_COL_LIMIT_STMT, entityName, fieldName, rowCount));
+ }
+
+ if (value.getClass().isAssignableFrom(String.class)) {
+ st.setString(1, String.valueOf(value));
+ } else if (value.getClass().isAssignableFrom(Long.class)) {
+ st.setLong(1, Long.parseLong(String.valueOf(value)));
}
var resultSet = st.executeQuery();
while (resultSet.next()){
- resultModels.add(MapperService.mapEntityToModel(resultSet, type));
+ resultModels.add(MapperService.mapResultToModel(resultSet, type));
}
} catch (SQLException e) {
throw new RuntimeException(e);
@@ -236,49 +272,93 @@ public class DerbyRepository implements IRepository {
return resultModels;
}
- @Override
- public void updateEntity(EntityEnum type, BaseEntityModel model) {
+ public <T extends BaseModel> void removeAllEntitiesByField(String fieldName, Object value, Class<T> type) {
try {
+
+ var entityName = type.getSimpleName().replace(Constants.ENTITY_CLASS_SUFFIX, "");
+
PreparedStatement st = _db.prepareStatement(
- String.format(UPDATE_ENTITY_WITH_COL_STMT, Constants.WORKERS_TABLE_NAME,
- Constants.ENTITY_NAME_COL,
- Constants.ENTITY_ADDR_COL,
- Constants.ENTITY_ID_COL));
- NodeEntityModel editModel = (NodeEntityModel) model;
+ String.format(DELETE_ENTITY_WITH_COL_STMT, entityName, fieldName));
- if (type == EntityEnum.COORDINATOR) {
- st = _db.prepareStatement(
- String.format(UPDATE_ENTITY_WITH_COL_STMT, Constants.COORDINATORS_TABLE_NAME,
- Constants.ENTITY_NAME_COL,
- Constants.ENTITY_ADDR_COL,
- Constants.ENTITY_ID_COL));
+ if (value.getClass().isAssignableFrom(String.class)) {
+ st.setString(1, String.valueOf(value));
+ } else if (value.getClass().isAssignableFrom(Long.class)) {
+ st.setLong(1, Long.parseLong(String.valueOf(value)));
}
- st.setString(1, editModel.getName());
- st.setString(2, editModel.getAddress());
- st.setLong(3, editModel.getId());
-
st.executeUpdate();
-
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
- public void removeEntity(EntityEnum type, Long id) {
- PreparedStatement st = null;
+ public <T extends BaseModel> void updateEntity(T model) {
+
try {
- if (type == EntityEnum.WORKER) {
- st = _db.prepareStatement(
- String.format(DELETE_ENTITY_WITH_COL_STMT, Constants.WORKERS_TABLE_NAME, Constants.ENTITY_ID_COL));
- st.setLong(1, id);
- } else {
- st = _db.prepareStatement(
- String.format(DELETE_ENTITY_WITH_COL_STMT, Constants.COORDINATORS_TABLE_NAME, Constants.ENTITY_ID_COL));
- st.setLong(1, id);
+ StringBuilder sb = new StringBuilder();
+
+ var entityName = model.getClass().getSimpleName().replace(Constants.ENTITY_CLASS_SUFFIX, "");
+
+ var fields = model.getClass().getFields();
+ var fieldsToChange = new ArrayList<Field>();
+ for (var field: fields) {
+
+ if (field.getName().equalsIgnoreCase(Constants.ENTITY_ID_COL)) {
+ continue;
+ }
+
+ if (field.getType().isAssignableFrom(String.class) ||
+ field.getType().isAssignableFrom(double.class) ||
+ field.getType().isAssignableFrom(Long.class) ||
+ field.getType().isAssignableFrom(int.class) ||
+ field.getType().isAssignableFrom(LocalDateTime.class)) {
+
+ if (field.get(model) != null) {
+ sb.append(String.format("%s = ?,", field.getName()));
+ fieldsToChange.add(field);
+ }
+ }
}
+
+ sb.replace(sb.length() - 1, sb.length(), "");
+
+ PreparedStatement st = _db.prepareStatement(String.format(UPDATE_ENTITY_WITH_COL_STMT, entityName, sb, Constants.ENTITY_ID_COL));
+
+ for (int i = 0; i < fieldsToChange.size(); i++) {
+ var field = fieldsToChange.get(i);
+
+ if (field.getType().isAssignableFrom(String.class)) {
+ st.setString(i + 1, String.valueOf(field.get(model)));
+ } else if (field.getType().isAssignableFrom(double.class)) {
+ st.setDouble(i + 1, (double) field.get(model));
+ } else if (field.getType().isAssignableFrom(Long.class) ||
+ field.getType().isAssignableFrom(int.class)) {
+ st.setLong(i + 1, (long) field.get(model));
+ } else if (field.getType().isAssignableFrom(LocalDateTime.class)) {
+ st.setTimestamp(i + 1, Timestamp.valueOf((LocalDateTime) field.get(model)));
+ }
+ }
+
+ st.setLong(fieldsToChange.size() + 1, model.id);
+
+ st.executeUpdate();
+
+ } catch (SQLException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public <T extends BaseModel> void removeEntity(Long id, Class<T> type) {
+ try {
+ var entityName = type.getSimpleName().replace(Constants.ENTITY_CLASS_SUFFIX, "");
+
+ PreparedStatement st = _db.prepareStatement(
+ String.format(DELETE_ENTITY_WITH_COL_STMT, entityName, Constants.ENTITY_ID_COL));
+
+ st.setLong(1, id);
st.executeUpdate();
} catch (SQLException e) {
throw new RuntimeException(e);
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
index dd683080e2..c4a0544f27 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/repositories/IRepository.java
@@ -20,19 +20,17 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseModel;
import java.util.List;
public interface IRepository {
- Long createEntity(EntityEnum type, BaseEntityModel model);
-
- BaseEntityModel getEntity(EntityEnum type, Long id);
-
- List<BaseEntityModel> getAllEntities(EntityEnum type);
-
- List<BaseEntityModel> getAllEntitiesByField(EntityEnum type, Object fieldValue);
- void updateEntity(EntityEnum type, BaseEntityModel model);
-
- void removeEntity(EntityEnum type, Long id);
+ <T extends BaseModel> Long createEntity(T model);
+ <T extends BaseModel> T getEntity(Long id, Class<T> type);
+ <T extends BaseModel> List<T> getAllEntities(Class<T> type);
+ <T extends BaseModel> List<T> getAllEntitiesByField(String fieldName, Object value, Class<T> type);
+ <T extends BaseModel> List<T> getAllEntitiesByField(String fieldName, Object value, Class<T> type, int rowCount);
+ <T extends BaseModel> void removeAllEntitiesByField(String fieldName, Object value, Class<T> type);
+ <T extends BaseModel> void updateEntity(T model);
+ <T extends BaseModel> void removeEntity(Long id, Class<T> type);
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/CoordinatorService.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/CoordinatorService.java
index 91137acaa2..98db405a14 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/CoordinatorService.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/CoordinatorService.java
@@ -19,33 +19,32 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.services;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.CoordinatorModel;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.DerbyRepository;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.EntityEnum;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.IRepository;
import java.util.List;
public class CoordinatorService {
- private static final IRepository _entityRepository = new DerbyRepository();
+ private static final IRepository entityRepository = new DerbyRepository();
- public void create(BaseEntityModel model) {
- _entityRepository.createEntity(EntityEnum.COORDINATOR, model);
+ public Long create(CoordinatorModel model) {
+ return entityRepository.createEntity(model);
}
- public void update(BaseEntityModel model) {
- _entityRepository.updateEntity(EntityEnum.COORDINATOR, model);
+ public void update(CoordinatorModel model) {
+ entityRepository.updateEntity(model);
}
public void remove(Long id) {
- _entityRepository.removeEntity(EntityEnum.COORDINATOR, id);
+ entityRepository.removeEntity(id, CoordinatorModel.class);
}
- public BaseEntityModel get(Long id) {
- return _entityRepository.getEntity(EntityEnum.COORDINATOR, id);
+ public CoordinatorModel get(Long id) {
+ return entityRepository.getEntity(id, CoordinatorModel.class);
}
- public List<BaseEntityModel> getAll() {
- return _entityRepository.getAllEntities(EntityEnum.COORDINATOR);
+ public List<CoordinatorModel> getAll() {
+ return entityRepository.getAllEntities(CoordinatorModel.class);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/MapperService.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/MapperService.java
index cd0efcc732..01b5c7122a 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/MapperService.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/MapperService.java
@@ -19,73 +19,64 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.services;
+import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.NodeEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.Request;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatsEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.Constants;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.EntityEnum;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.Request;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
public class MapperService {
- public static BaseEntityModel getModelFromBody(Request request) {
+ public static <T extends BaseModel> T getModelFromBody(Request request, Class<T> classType) {
ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
try {
- return mapper.readValue(request.getBody(), NodeEntityModel.class);
+ if (!request.getBody().isEmpty() && !request.getBody().isBlank()) {
+ return mapper.readValue(request.getBody(), classType);
+ }
+
+ return classType.getDeclaredConstructor().newInstance();
}
- catch (IOException e) {
+ catch (IOException | InvocationTargetException | IllegalAccessException | InstantiationException |
+ NoSuchMethodException e) {
throw new RuntimeException(e);
}
}
- public static BaseEntityModel mapEntityToModel(ResultSet resultSet, EntityEnum targetModel) {
+ public static <T extends BaseModel> T mapResultToModel(ResultSet resultSet, Class<T> classType) {
try {
- if (targetModel != EntityEnum.WORKER_STATS) {
- NodeEntityModel tmpModel = new NodeEntityModel();
- for (int column = 1; column <= resultSet.getMetaData().getColumnCount(); column++) {
- if (resultSet.getMetaData().getColumnType(column) == Types.INTEGER) {
- tmpModel.setId(resultSet.getLong(column));
- }
+ var result = classType.getDeclaredConstructor().newInstance();
+ var fields = result.getClass().getFields();
- if (resultSet.getMetaData().getColumnType(column) == Types.VARCHAR) {
- if (resultSet.getMetaData().getColumnName(column).equalsIgnoreCase(Constants.ENTITY_NAME_COL)) {
- tmpModel.setName(resultSet.getString(column));
- } else if (resultSet.getMetaData().getColumnName(column).equalsIgnoreCase(Constants.ENTITY_ADDR_COL)) {
- tmpModel.setAddress(resultSet.getString(column));
- }
- }
- }
- return tmpModel;
- } else {
- StatsEntityModel tmpModel = new StatsEntityModel();
+ for (int column = 1; column <= resultSet.getMetaData().getColumnCount(); column++) {
- for (int column = 1; column <= resultSet.getMetaData().getColumnCount(); column++) {
+ var colName = resultSet.getMetaData().getColumnName(column);
- if (resultSet.getMetaData().getColumnType(column) == Types.VARCHAR) {
- if (resultSet.getMetaData().getColumnName(column).equalsIgnoreCase(Constants.ENTITY_TRAFFIC_COL)) {
- tmpModel.setTransferredBytes(resultSet.getString(column));
- } else if (resultSet.getMetaData().getColumnName(column).equalsIgnoreCase(Constants.ENTITY_HEAVY_HITTERS_COL)) {
- tmpModel.setHeavyHitterInstructions(resultSet.getString(column));
- }
- } else {
- if (resultSet.getMetaData().getColumnName(column).equalsIgnoreCase(Constants.ENTITY_CPU_COL)) {
- tmpModel.setCPUUsage(resultSet.getDouble(column));
- } else if (resultSet.getMetaData().getColumnName(column).equalsIgnoreCase(Constants.ENTITY_MEM_COL)) {
- tmpModel.setMemoryUsage(resultSet.getDouble(column));
+ for (var field: fields) {
+ var fieldName = field.getName();
+ if (colName.equalsIgnoreCase(fieldName)) {
+ if (resultSet.getMetaData().getColumnType(column) == Types.VARCHAR) {
+ result.getClass().getField(fieldName).set(result, resultSet.getString(column));
+ } else if (resultSet.getMetaData().getColumnType(column) == Types.DOUBLE) {
+ result.getClass().getField(fieldName).set(result, resultSet.getDouble(column));
+ } else if (resultSet.getMetaData().getColumnType(column) == Types.INTEGER) {
+ result.getClass().getField(fieldName).set(result, resultSet.getLong(column));
+ } else if (resultSet.getMetaData().getColumnType(column) == Types.TIMESTAMP) {
+ result.getClass().getField(fieldName).set(result, resultSet.getTimestamp(column).toLocalDateTime());
}
}
}
-
- return tmpModel;
}
- } catch (SQLException e) {
+
+ return result;
+ } catch (SQLException | NoSuchMethodException | InvocationTargetException | InstantiationException |
+ IllegalAccessException | NoSuchFieldException e) {
throw new RuntimeException(e);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatisticsService.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatisticsService.java
new file mode 100644
index 0000000000..0b05effce9
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatisticsService.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.controlprogram.federated.monitoring.services;
+
+import org.apache.sysds.api.DMLScript;
+import org.apache.sysds.runtime.DMLRuntimeException;
+import org.apache.sysds.runtime.controlprogram.federated.FederatedData;
+import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest;
+import org.apache.sysds.runtime.controlprogram.federated.FederatedResponse;
+import org.apache.sysds.runtime.controlprogram.federated.FederatedStatistics;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.*;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.Constants;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.DerbyRepository;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.IRepository;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class StatisticsService {
+
+ private static final IRepository entityRepository = new DerbyRepository();
+
+ public StatisticsModel getAll(Long workerId, StatisticsOptions options) {
+ var stats = new StatisticsModel();
+
+ if (options.utilization) {
+ stats.utilization = entityRepository.getAllEntitiesByField(Constants.ENTITY_WORKER_ID_COL, workerId, UtilizationModel.class, options.rowCount);
+ }
+
+ if (options.traffic) {
+ stats.traffic = entityRepository.getAllEntitiesByField(Constants.ENTITY_WORKER_ID_COL, workerId, TrafficModel.class, options.rowCount);
+ }
+
+ if (options.events) {
+ stats.events = entityRepository.getAllEntitiesByField(Constants.ENTITY_WORKER_ID_COL, workerId, EventModel.class, options.rowCount);
+
+ for (var event: stats.events) {
+ event.setCoordinatorName(entityRepository.getEntity(event.coordinatorId, CoordinatorModel.class).name);
+
+ event.stages = entityRepository.getAllEntitiesByField(Constants.ENTITY_EVENT_ID_COL, event.id, EventStageModel.class);
+ }
+ }
+
+ if (options.dataObjects) {
+ stats.dataObjects = entityRepository.getAllEntitiesByField(Constants.ENTITY_WORKER_ID_COL, workerId, DataObjectModel.class);
+ }
+
+ if (options.requests) {
+ stats.requests = entityRepository.getAllEntitiesByField(Constants.ENTITY_WORKER_ID_COL, workerId, RequestModel.class);
+ }
+
+ return stats;
+ }
+
+ public static StatisticsModel getWorkerStatistics(Long id, String address) {
+ StatisticsModel parsedStats = null;
+
+ try {
+ FederatedResponse statisticsResponse = null;
+
+ var statisticsResponseFuture = sendStatisticsRequest(address);
+
+ if (statisticsResponseFuture != null) {
+ statisticsResponse = statisticsResponseFuture.get();
+ }
+
+ if (statisticsResponse != null && statisticsResponse.isSuccessful()) {
+ FederatedStatistics.FedStatsCollection aggFedStats = new FederatedStatistics.FedStatsCollection();
+
+ Object[] tmp = statisticsResponse.getData();
+ if(tmp[0] instanceof FederatedStatistics.FedStatsCollection)
+ aggFedStats.aggregate((FederatedStatistics.FedStatsCollection)tmp[0]);
+
+ parsedStats = parseStatistics(id, aggFedStats);
+ }
+ } catch(DMLRuntimeException dre) {
+ // silently ignore -> caused by offline federated workers
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ return parsedStats;
+ }
+
+ private static StatisticsModel parseStatistics(Long workerId, FederatedStatistics.FedStatsCollection aggFedStats) {
+ var utilization = aggFedStats.utilization;
+ var traffic = aggFedStats.coordinatorsTrafficBytes;
+ var events = aggFedStats.workerEvents;
+ var dataObjects = aggFedStats.workerDataObjects;
+ var requests = aggFedStats.workerRequests;
+
+ utilization.workerId = workerId;
+ traffic.forEach(t -> t.workerId = workerId);
+ dataObjects.forEach(o -> o.workerId = workerId);
+
+
+ for (var event: events) {
+ event.workerId = workerId;
+
+ setCoordinatorId(event);
+ }
+
+ for (var trafficEntry: traffic) {
+ trafficEntry.workerId = workerId;
+
+ setCoordinatorId(trafficEntry);
+ }
+
+ for (var request: requests) {
+ request.workerId = workerId;
+
+ setCoordinatorId(request);
+ }
+
+ return new StatisticsModel(List.of(utilization), traffic, events, dataObjects, requests);
+ }
+
+ private static void setCoordinatorId(CoordinatorConnectionModel entity) {
+ List<CoordinatorModel> coordinators = new ArrayList<>();
+ var monitoringKey = entity.getCoordinatorHostId();
+
+ if (monitoringKey != null) {
+ coordinators = entityRepository.getAllEntitiesByField(Constants.ENTITY_MONITORING_KEY_COL, monitoringKey, CoordinatorModel.class);
+ }
+
+ if (!coordinators.isEmpty()) {
+ entity.coordinatorId = coordinators.get(0).id;
+ } else {
+ entity.coordinatorId = -1L;
+ }
+ }
+
+ private static Future<FederatedResponse> sendStatisticsRequest(String address) {
+ Future<FederatedResponse> result = null;
+
+ final Pattern pattern = Pattern.compile("(.*://)?([A-Za-z0-9\\-\\.]+)(:[0-9]+)?(.*)");
+ final Matcher matcher = pattern.matcher(address);
+
+ if (matcher.find()) {
+ String host = matcher.group(2);
+ String portStr = matcher.group(3);
+ int port = 80;
+
+ if (portStr != null && !portStr.isBlank() && !portStr.isEmpty())
+ port = Integer.parseInt(portStr.replace(":", ""));
+
+ InetSocketAddress isa = new InetSocketAddress(host, port);
+ FederatedRequest frUDF = new FederatedRequest(FederatedRequest.RequestType.EXEC_UDF, -1,
+ new FederatedStatistics.FedStatsCollectFunction());
+
+ try {
+ result = FederatedData.executeFederatedOperation(isa, frUDF);
+ } catch(DMLRuntimeException dre) {
+ throw dre; // caused by offline federated workers
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ return result;
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatsService.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatsService.java
deleted file mode 100644
index 565f1b2712..0000000000
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/StatsService.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysds.runtime.controlprogram.federated.monitoring.services;
-
-import org.apache.sysds.runtime.DMLRuntimeException;
-import org.apache.sysds.runtime.controlprogram.federated.FederatedData;
-import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest;
-import org.apache.sysds.runtime.controlprogram.federated.FederatedResponse;
-import org.apache.sysds.runtime.controlprogram.federated.FederatedStatistics;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatsEntityModel;
-
-import java.net.InetSocketAddress;
-import java.util.concurrent.Future;
-
-public class StatsService {
- public static BaseEntityModel getWorkerStatistics(Long id, String address) {
- StatsEntityModel parsedStats = null;
-
- try {
- var statisticsResponse = sendStatisticsRequest(address).get();
-
- if (statisticsResponse.isSuccessful()) {
- FederatedStatistics.FedStatsCollection aggFedStats = new FederatedStatistics.FedStatsCollection();
-
- Object[] tmp = statisticsResponse.getData();
- if(tmp[0] instanceof FederatedStatistics.FedStatsCollection)
- aggFedStats.aggregate((FederatedStatistics.FedStatsCollection)tmp[0]);
-
- parsedStats = new StatsEntityModel(
- id, aggFedStats.cpuUsage, aggFedStats.memoryUsage,
- aggFedStats.heavyHitters, aggFedStats.coordinatorsTrafficBytes);
- }
- } catch(DMLRuntimeException dre) {
- // silently ignore -> caused by offline federated workers
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- return parsedStats;
- }
-
- private static Future<FederatedResponse> sendStatisticsRequest(String address) {
- Future<FederatedResponse> result = null;
- String host = address.split(":")[0];
- int port = Integer.parseInt(address.split(":")[1]);
-
- InetSocketAddress isa = new InetSocketAddress(host, port);
- FederatedRequest frUDF = new FederatedRequest(FederatedRequest.RequestType.EXEC_UDF, -1,
- new FederatedStatistics.FedStatsCollectFunction());
- try {
- result = FederatedData.executeFederatedOperation(isa, frUDF);
- } catch(DMLRuntimeException dre) {
- throw dre; // caused by offline federated workers
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- return result;
- }
-}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/WorkerService.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/WorkerService.java
index 82845177c3..854b804c14 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/WorkerService.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/services/WorkerService.java
@@ -19,14 +19,15 @@
package org.apache.sysds.runtime.controlprogram.federated.monitoring.services;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.BaseEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.NodeEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatsEntityModel;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.DataObjectModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.RequestModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.Constants;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.DerbyRepository;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.EntityEnum;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.IRepository;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -35,88 +36,129 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
public class WorkerService {
- private static final IRepository _entityRepository = new DerbyRepository();
- private static final Map<Long, String> _cachedWorkers = new HashMap<>();
+ private static final IRepository entityRepository = new DerbyRepository();
+ // { workerId, { workerAddress, workerStatus } }
+ private static final Map<Long, Pair<String, Boolean>> cachedWorkers = new HashMap<>();
public WorkerService() {
- updateCachedWorkers(null);
-
ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
executor.scheduleAtFixedRate(syncWorkerStatisticsWithDB(), 0, 3, TimeUnit.SECONDS);
}
- public void create(BaseEntityModel model) {
- long id = _entityRepository.createEntity(EntityEnum.WORKER, model);
+ public Long create(WorkerModel model) {
- var modelEntity = (NodeEntityModel) model;
+ long id = entityRepository.createEntity(model);
+ model.id = id;
- _cachedWorkers.putIfAbsent(id, modelEntity.getAddress());
- }
+ updateCachedWorkers(List.of(model), false);
- public void update(BaseEntityModel model) {
- _entityRepository.updateEntity(EntityEnum.WORKER, model);
+ return id;
}
- public void remove(Long id) {
- _entityRepository.removeEntity(EntityEnum.WORKER, id);
+ public void update(WorkerModel model) {
+ entityRepository.updateEntity(model);
- _cachedWorkers.remove(id);
+ updateCachedWorkers(List.of(model), false);
}
- public BaseEntityModel get(Long id) {
- var model = (NodeEntityModel) _entityRepository.getEntity(EntityEnum.WORKER, id);
- var stats = (List<BaseEntityModel>) _entityRepository.getAllEntitiesByField(EntityEnum.WORKER_STATS, id);
-
- updateCachedWorkers(null);
-
- model.setStats(stats);
+ public void remove(Long id) {
+ entityRepository.removeEntity(id, WorkerModel.class);
- return model;
+ updateCachedWorkers(List.of(new WorkerModel(id)), true);
}
- public List<BaseEntityModel> getAll() {
- var workersRaw = _entityRepository.getAllEntities(EntityEnum.WORKER);
- var workersResult = new ArrayList<BaseEntityModel>();
+ public WorkerModel get(Long id) {
+ var worker = entityRepository.getEntity(id, WorkerModel.class);
- updateCachedWorkers(workersRaw);
+ updateCachedWorkers(List.of(worker), false);
- for (var worker: workersRaw) {
- var workerModel = (NodeEntityModel) worker;
- var stats = (List<BaseEntityModel>) _entityRepository.getAllEntitiesByField(EntityEnum.WORKER_STATS, workerModel.getId());
+ return worker;
+ }
- workerModel.setStats(stats);
+ public List<WorkerModel> getAll() {
+ var workers = entityRepository.getAllEntities(WorkerModel.class);
- workersResult.add(workerModel);
- }
+ updateCachedWorkers(workers, false);
- return workersResult;
+ return workers;
}
- private void updateCachedWorkers(List<BaseEntityModel> workersRaw) {
- List<BaseEntityModel> workersBaseModel = workersRaw;
-
- if (workersBaseModel == null) {
- workersBaseModel = getAll();
- }
+ public Boolean getWorkerOnlineStatus(Long workerId) {
+ return cachedWorkers.get(workerId).getRight();
+ }
- for(var workerBaseModel : workersBaseModel) {
- var worker = (NodeEntityModel) workerBaseModel;
+ private static synchronized void updateCachedWorkers(List<WorkerModel> workers, boolean removeList) {
- _cachedWorkers.putIfAbsent(worker.getId(), worker.getAddress());
+ if (removeList) {
+ for (var worker: workers) {
+ cachedWorkers.remove(worker.id);
+ }
+ } else {
+ for (var worker: workers) {
+ if (!cachedWorkers.containsKey(worker.id)) {
+ cachedWorkers.put(worker.id, new MutablePair<>(worker.address, false));
+ } else {
+ var oldPair = cachedWorkers.get(worker.id);
+ cachedWorkers.replace(worker.id, new MutablePair<>(worker.address, oldPair.getRight()));
+ }
+ }
}
}
private static Runnable syncWorkerStatisticsWithDB() {
return () -> {
- for(Map.Entry<Long, String> entry : _cachedWorkers.entrySet()) {
+ for(Map.Entry<Long, Pair<String, Boolean>> entry : cachedWorkers.entrySet()) {
Long id = entry.getKey();
- String address = entry.getValue();
+ String address = entry.getValue().getLeft();
- var stats = (StatsEntityModel) StatsService.getWorkerStatistics(id, address);
+ var stats = StatisticsService.getWorkerStatistics(id, address);
if (stats != null) {
- _entityRepository.createEntity(EntityEnum.WORKER_STATS, stats);
+
+ cachedWorkers.get(id).setValue(true);
+
+ if (stats.utilization != null) {
+ entityRepository.createEntity(stats.utilization.get(0));
+ }
+ if (stats.traffic != null) {
+ for (var trafficEntity: stats.traffic) {
+ if (trafficEntity.coordinatorId > 0) {
+ entityRepository.createEntity(trafficEntity);
+ }
+ }
+ }
+ if (stats.events != null) {
+ for (var eventEntity: stats.events) {
+ if (eventEntity.coordinatorId > 0) {
+ var eventId = entityRepository.createEntity(eventEntity);
+
+ for (var stageEntity: eventEntity.stages) {
+ stageEntity.eventId = eventId;
+
+ entityRepository.createEntity(stageEntity);
+ }
+ }
+ }
+ }
+ if (stats.dataObjects != null) {
+ entityRepository.removeAllEntitiesByField(Constants.ENTITY_WORKER_ID_COL, id, DataObjectModel.class);
+
+ for (var dataObjectEntity: stats.dataObjects) {
+ entityRepository.createEntity(dataObjectEntity);
+ }
+ }
+ if (stats.requests != null) {
+ entityRepository.removeAllEntitiesByField(Constants.ENTITY_WORKER_ID_COL, id, RequestModel.class);
+
+ for (var requestEntity: stats.requests) {
+ if (requestEntity.coordinatorId > 0) {
+ entityRepository.createEntity(requestEntity);
+ }
+ }
+ }
+ } else {
+ cachedWorkers.get(id).setValue(false);
}
}
};
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedCoordinatorIntegrationCRUDTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedCoordinatorIntegrationCRUDTest.java
index c6612f2cb9..d3cc095034 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedCoordinatorIntegrationCRUDTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedCoordinatorIntegrationCRUDTest.java
@@ -21,8 +21,7 @@ package org.apache.sysds.test.functions.federated.monitoring;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpStatus;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.NodeEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.EntityEnum;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
import org.apache.sysds.test.TestConfiguration;
import org.apache.sysds.test.TestUtils;
import org.junit.Assert;
@@ -44,7 +43,7 @@ public class FederatedCoordinatorIntegrationCRUDTest extends FederatedMonitoring
@Test
public void testCoordinatorAddedForMonitoring() {
- var addedCoordinators = addEntities(EntityEnum.COORDINATOR,1);
+ var addedCoordinators = addEntities(1);
var firstCoordinatorStatus = addedCoordinators.get(0).statusCode();
Assert.assertEquals("Added coordinator status code", HttpStatus.SC_OK, firstCoordinatorStatus);
@@ -53,10 +52,10 @@ public class FederatedCoordinatorIntegrationCRUDTest extends FederatedMonitoring
@Test
@Ignore
public void testCoordinatorRemovedFromMonitoring() {
- addEntities(EntityEnum.COORDINATOR,2);
- var statusCode = removeEntity(EntityEnum.COORDINATOR,1L).statusCode();
+ addEntities(2);
+ var statusCode = removeEntity(1L).statusCode();
- var getAllCoordinatorsResponse = getEntities(EntityEnum.COORDINATOR);
+ var getAllCoordinatorsResponse = getEntities();
var numReturnedCoordinators = StringUtils.countMatches(getAllCoordinatorsResponse.body().toString(), "id");
Assert.assertEquals("Removed coordinator status code", HttpStatus.SC_OK, statusCode);
@@ -66,13 +65,13 @@ public class FederatedCoordinatorIntegrationCRUDTest extends FederatedMonitoring
@Test
@Ignore
public void testCoordinatorDataUpdated() {
- addEntities(EntityEnum.COORDINATOR,3);
- var newCoordinatorData = new NodeEntityModel(1L, "NonExistentName", "nonexistent.address");
+ addEntities(3);
+ var newCoordinatorData = new WorkerModel(1L, "NonExistentName", "nonexistent.address");
- var editedCoordinator = updateEntity(EntityEnum.COORDINATOR, newCoordinatorData);
+ var editedCoordinator = updateEntity(newCoordinatorData);
- var getAllCoordinatorsResponse = getEntities(EntityEnum.COORDINATOR);
- var numCoordinatorsNewData = StringUtils.countMatches(getAllCoordinatorsResponse.body().toString(), newCoordinatorData.getName());
+ var getAllCoordinatorsResponse = getEntities();
+ var numCoordinatorsNewData = StringUtils.countMatches(getAllCoordinatorsResponse.body().toString(), newCoordinatorData.name);
Assert.assertEquals("Updated coordinator status code", HttpStatus.SC_OK, editedCoordinator.statusCode());
Assert.assertEquals("Updated coordinators num", 1, numCoordinatorsNewData);
@@ -82,14 +81,14 @@ public class FederatedCoordinatorIntegrationCRUDTest extends FederatedMonitoring
@Ignore
public void testCorrectAmountAddedCoordinatorsForMonitoring() {
int numCoordinators = 3;
- var addedCoordinators = addEntities(EntityEnum.COORDINATOR, numCoordinators);
+ var addedCoordinators = addEntities(numCoordinators);
for (int i = 0; i < numCoordinators; i++) {
var coordinatorStatus = addedCoordinators.get(i).statusCode();
Assert.assertEquals("Added coordinator status code", HttpStatus.SC_OK, coordinatorStatus);
}
- var getAllCoordinatorsResponse = getEntities(EntityEnum.COORDINATOR);
+ var getAllCoordinatorsResponse = getEntities();
var numReturnedCoordinators = StringUtils.countMatches(getAllCoordinatorsResponse.body().toString(), "id");
Assert.assertEquals("Amount of coordinators to get", numCoordinators, numReturnedCoordinators);
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedMonitoringTestBase.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedMonitoringTestBase.java
index 4206151686..5d611d6388 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedMonitoringTestBase.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedMonitoringTestBase.java
@@ -20,8 +20,7 @@
package org.apache.sysds.test.functions.federated.monitoring;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.NodeEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.EntityEnum;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
import org.apache.sysds.test.functions.federated.multitenant.MultiTenantTestBase;
import org.junit.After;
@@ -63,22 +62,17 @@ public abstract class FederatedMonitoringTestBase extends MultiTenantTestBase {
monitoringProcess = startLocalFedMonitoring(monitoringPort, addArgs);
}
- protected List<HttpResponse<?>> addEntities(EntityEnum type, int count) {
+ protected List<HttpResponse<?>> addEntities(int count) {
String uriStr = MAIN_URI + ":" + monitoringPort + WORKER_MAIN_PATH;
String name = "Worker";
- if (type == EntityEnum.COORDINATOR) {
- uriStr = MAIN_URI + ":" + monitoringPort + COORDINATOR_MAIN_PATH;
- name = "Coordinator";
- }
-
List<HttpResponse<?>> responses = new ArrayList<>();
try {
ObjectMapper objectMapper = new ObjectMapper();
for (int i = 0; i < count; i++) {
String requestBody = objectMapper
.writerWithDefaultPrettyPrinter()
- .writeValueAsString(new NodeEntityModel((i + 1L), name, "localhost"));
+ .writeValueAsString(new WorkerModel((i + 1L), name, "localhost"));
var client = HttpClient.newHttpClient();
var request = HttpRequest.newBuilder(URI.create(uriStr))
.header("accept", "application/json")
@@ -94,18 +88,14 @@ public abstract class FederatedMonitoringTestBase extends MultiTenantTestBase {
}
}
- protected HttpResponse<?> updateEntity(EntityEnum type, NodeEntityModel editModel) {
+ protected HttpResponse<?> updateEntity(WorkerModel editModel) {
String uriStr = MAIN_URI + ":" + monitoringPort + WORKER_MAIN_PATH;
- if (type == EntityEnum.COORDINATOR) {
- uriStr = MAIN_URI + ":" + monitoringPort + COORDINATOR_MAIN_PATH;
- }
-
try {
ObjectMapper objectMapper = new ObjectMapper();
String requestBody = objectMapper
.writerWithDefaultPrettyPrinter()
- .writeValueAsString(new NodeEntityModel(editModel.getId(), editModel.getName(), editModel.getAddress()));
+ .writeValueAsString(new WorkerModel(editModel.id, editModel.name, editModel.address));
var client = HttpClient.newHttpClient();
var request = HttpRequest.newBuilder(URI.create(uriStr))
.header("accept", "application/json")
@@ -119,13 +109,9 @@ public abstract class FederatedMonitoringTestBase extends MultiTenantTestBase {
}
}
- protected HttpResponse<?> removeEntity(EntityEnum type, Long id) {
+ protected HttpResponse<?> removeEntity(Long id) {
String uriStr = MAIN_URI + ":" + monitoringPort + WORKER_MAIN_PATH + "/" + id;
- if (type == EntityEnum.COORDINATOR) {
- uriStr = MAIN_URI + ":" + monitoringPort + COORDINATOR_MAIN_PATH + "/" + id;
- }
-
try {
var client = HttpClient.newHttpClient();
var request = HttpRequest.newBuilder(URI.create(uriStr))
@@ -140,13 +126,9 @@ public abstract class FederatedMonitoringTestBase extends MultiTenantTestBase {
}
}
- protected HttpResponse<?> getEntities(EntityEnum type) {
+ protected HttpResponse<?> getEntities() {
String uriStr = MAIN_URI + ":" + monitoringPort + WORKER_MAIN_PATH;
- if (type == EntityEnum.COORDINATOR) {
- uriStr = MAIN_URI + ":" + monitoringPort + COORDINATOR_MAIN_PATH;
- }
-
try {
var client = HttpClient.newHttpClient();
var request = HttpRequest.newBuilder(URI.create(uriStr))
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerIntegrationCRUDTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerIntegrationCRUDTest.java
index 2282c06871..b70c9c11c2 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerIntegrationCRUDTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerIntegrationCRUDTest.java
@@ -21,8 +21,7 @@ package org.apache.sysds.test.functions.federated.monitoring;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpStatus;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.NodeEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.EntityEnum;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
import org.apache.sysds.test.TestConfiguration;
import org.apache.sysds.test.TestUtils;
import org.junit.Assert;
@@ -44,7 +43,7 @@ public class FederatedWorkerIntegrationCRUDTest extends FederatedMonitoringTestB
@Test
public void testWorkerAddedForMonitoring() {
- var addedWorkers = addEntities(EntityEnum.WORKER,1);
+ var addedWorkers = addEntities(1);
var firstWorkerStatus = addedWorkers.get(0).statusCode();
Assert.assertEquals("Added worker status code", HttpStatus.SC_OK, firstWorkerStatus);
@@ -53,10 +52,10 @@ public class FederatedWorkerIntegrationCRUDTest extends FederatedMonitoringTestB
@Test
@Ignore
public void testWorkerRemovedFromMonitoring() {
- addEntities(EntityEnum.WORKER,2);
- var statusCode = removeEntity(EntityEnum.WORKER,1L).statusCode();
+ addEntities(2);
+ var statusCode = removeEntity(1L).statusCode();
- var getAllWorkersResponse = getEntities(EntityEnum.WORKER);
+ var getAllWorkersResponse = getEntities();
var numReturnedWorkers = StringUtils.countMatches(getAllWorkersResponse.body().toString(), "id");
Assert.assertEquals("Removed worker status code", HttpStatus.SC_OK, statusCode);
@@ -66,13 +65,13 @@ public class FederatedWorkerIntegrationCRUDTest extends FederatedMonitoringTestB
@Test
@Ignore
public void testWorkerDataUpdated() {
- addEntities(EntityEnum.WORKER,3);
- var newWorkerData = new NodeEntityModel(1L, "NonExistentName", "nonexistent.address");
+ addEntities(3);
+ var newWorkerData = new WorkerModel(1L, "NonExistentName", "nonexistent.address");
- var editedWorker = updateEntity(EntityEnum.WORKER, newWorkerData);
+ var editedWorker = updateEntity(newWorkerData);
- var getAllWorkersResponse = getEntities(EntityEnum.WORKER);
- var numWorkersNewData = StringUtils.countMatches(getAllWorkersResponse.body().toString(), newWorkerData.getName());
+ var getAllWorkersResponse = getEntities();
+ var numWorkersNewData = StringUtils.countMatches(getAllWorkersResponse.body().toString(), newWorkerData.name);
Assert.assertEquals("Updated worker status code", HttpStatus.SC_OK, editedWorker.statusCode());
Assert.assertEquals("Updated workers num", 1, numWorkersNewData);
@@ -82,14 +81,14 @@ public class FederatedWorkerIntegrationCRUDTest extends FederatedMonitoringTestB
@Ignore
public void testCorrectAmountAddedWorkersForMonitoring() {
int numWorkers = 3;
- var addedWorkers = addEntities(EntityEnum.WORKER, numWorkers);
+ var addedWorkers = addEntities(numWorkers);
for (int i = 0; i < numWorkers; i++) {
var workerStatus = addedWorkers.get(i).statusCode();
Assert.assertEquals("Added worker status code", HttpStatus.SC_OK, workerStatus);
}
- var getAllWorkersResponse = getEntities(EntityEnum.WORKER);
+ var getAllWorkersResponse = getEntities();
var numReturnedWorkers = StringUtils.countMatches(getAllWorkersResponse.body().toString(), "id");
Assert.assertEquals("Amount of workers to get", numWorkers, numReturnedWorkers);
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerStatisticsTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerStatisticsTest.java
index 2eaa3a6232..af39724f70 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerStatisticsTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedWorkerStatisticsTest.java
@@ -19,9 +19,13 @@
package org.apache.sysds.test.functions.federated.monitoring;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.NodeEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatsEntityModel;
-import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.StatsService;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.EventModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.EventStageModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatisticsModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.StatisticsOptions;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.models.WorkerModel;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.repositories.DerbyRepository;
+import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.StatisticsService;
import org.apache.sysds.runtime.controlprogram.federated.monitoring.services.WorkerService;
import org.apache.sysds.test.TestConfiguration;
import org.apache.sysds.test.TestUtils;
@@ -36,6 +40,7 @@ public class FederatedWorkerStatisticsTest extends FederatedMonitoringTestBase {
private static int[] workerPorts;
private final WorkerService workerMonitoringService = new WorkerService();
+ private final StatisticsService statisticsMonitoringService = new StatisticsService();
@Override
public void setUp() {
@@ -47,27 +52,42 @@ public class FederatedWorkerStatisticsTest extends FederatedMonitoringTestBase {
@Test
public void testWorkerStatisticsParsedCorrectly() {
- var model = (StatsEntityModel) StatsService.getWorkerStatistics(1L, "localhost:" + workerPorts[0]);
+ var model = (StatisticsModel) StatisticsService.getWorkerStatistics(1L, "localhost:" + workerPorts[0]);
Assert.assertNotNull("Stats parsed correctly", model);
- Assert.assertNotEquals("CPU stats parsed correctly", 0, model.getCPUUsage());
- Assert.assertNotEquals("Memory Stats parsed correctly", 0, model.getMemoryUsage());
+ Assert.assertNotEquals("Utilization stats parsed correctly", 0, model.utilization.size());
}
@Test
public void testWorkerStatisticsReturnedForMonitoring() {
- workerMonitoringService.create(new NodeEntityModel(1L, "Worker", "localhost:" + workerPorts[0]));
+ workerMonitoringService.create(new WorkerModel(1L, "Worker", "localhost:" + workerPorts[0]));
- var model = (NodeEntityModel) workerMonitoringService.get(1L);
+ var model = workerMonitoringService.get(1L);
- Assert.assertNotNull("Stats field of model contains worker statistics", model.getStats());
+ Assert.assertNotNull("Stats field of model contains worker statistics", model);
}
@Test
public void testNonExistentWorkerStatistics() {
- workerMonitoringService.create(new NodeEntityModel(1L, "Worker", "not-running.address"));
- var model = (NodeEntityModel) workerMonitoringService.get(1L);
+ var bla = new EventModel(1L, -1L);
+ var derby = new DerbyRepository();
- Assert.assertEquals("Stats field of model contains worker statistics", 0, model.getStats().size());
+ var in1 = derby.createEntity(bla);
+ var in2 = derby.createEntity(bla);
+ var in3 = derby.createEntity(bla);
+ var in4 = derby.createEntity(bla);
+
+ var shit = derby.getEntity(in3, EventModel.class);
+
+ var stage = new EventStageModel();
+
+
+ workerMonitoringService.create(new WorkerModel(1L, "Worker", "localhost:8001"));
+ var options = new StatisticsOptions();
+ options.utilization = true;
+
+ var stats = statisticsMonitoringService.getAll(1L, options);
+
+ Assert.assertEquals("Utilization field of model contains worker statistics", 0, stats.utilization.size());
}
}