You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by go...@apache.org on 2021/06/22 14:39:29 UTC

[incubator-streampipes] branch edge-extensions updated: Introduced MQTT Logging

This is an automated email from the ASF dual-hosted git repository.

gomm pushed a commit to branch edge-extensions
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git


The following commit(s) were added to refs/heads/edge-extensions by this push:
     new 094bb68  Introduced MQTT Logging
094bb68 is described below

commit 094bb6836d11b7723718afa27bf9dcfa2a2f8748
Author: daniel-gomm <da...@outlook.de>
AuthorDate: Tue Jun 22 16:36:35 2021 +0200

    Introduced MQTT Logging
---
 streampipes-logging/pom.xml                        |  5 +++
 .../logging/evaluation/EvaluationLogger.java       | 41 ++++++++++++++++++++--
 .../controller/api/InvocableEntityResource.java    | 11 +++---
 .../offloading/OffloadingPolicyManager.java        | 10 +++---
 .../EdgeExtensionsGenericPerformanceTest.java      |  7 ++--
 .../streampipes/performance/TestFactory.java       |  8 +++++
 .../performance/performancetest/GenericTest.java   | 14 +++++---
 7 files changed, 74 insertions(+), 22 deletions(-)

diff --git a/streampipes-logging/pom.xml b/streampipes-logging/pom.xml
index d648b2f..f2bb53c 100644
--- a/streampipes-logging/pom.xml
+++ b/streampipes-logging/pom.xml
@@ -34,5 +34,10 @@
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.fusesource.mqtt-client</groupId>
+            <artifactId>mqtt-client</artifactId>
+            <version>1.12</version>
+        </dependency>
     </dependencies>
 </project>
\ No newline at end of file
diff --git a/streampipes-logging/src/main/java/org/apache/streampipes/logging/evaluation/EvaluationLogger.java b/streampipes-logging/src/main/java/org/apache/streampipes/logging/evaluation/EvaluationLogger.java
index 501d4e0..2d2d2bc 100644
--- a/streampipes-logging/src/main/java/org/apache/streampipes/logging/evaluation/EvaluationLogger.java
+++ b/streampipes-logging/src/main/java/org/apache/streampipes/logging/evaluation/EvaluationLogger.java
@@ -17,20 +17,26 @@
  */
 package org.apache.streampipes.logging.evaluation;
 
+import org.fusesource.mqtt.client.BlockingConnection;
+import org.fusesource.mqtt.client.MQTT;
+import org.fusesource.mqtt.client.QoS;
+
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
 public class EvaluationLogger {
 
-    private static String FILE_NAME = "evaluation-logs.csv";
-
     private final File file;
     private String buffer = "";
     private static EvaluationLogger instance = null;
+    private final MQTT mqtt;
+    private final BlockingConnection connection;
 
     public static EvaluationLogger getInstance(){
         if(instance==null) instance = new EvaluationLogger();
@@ -39,9 +45,12 @@ public class EvaluationLogger {
 
     private EvaluationLogger(){
         String filepath = System.getenv("SP_LOGGING_FILE_PATH");
+        String logging_host = System.getenv("SP_LOGGING_MQTT_HOST");
+        int logging_port = Integer.parseInt(System.getenv("SP_LOGGING_MQTT_PORT"));
         if(filepath==null) throw new RuntimeException("No Logging Location provided.");
         if(!filepath.endsWith("/")) filepath = filepath + "/";
         String timePrefix = new SimpleDateFormat("yyyy-MM-dd-HHmmss").format(new Date());
+        String FILE_NAME = "evaluation-logs.csv";
         String fileWithPath = new StringBuilder()
                 .append(filepath)
                 .append(timePrefix)
@@ -57,15 +66,32 @@ public class EvaluationLogger {
         } catch (IOException e) {
             e.printStackTrace();
         }
+
+        mqtt = new MQTT();
+        try {
+            mqtt.setHost(logging_host, logging_port);
+        } catch (URISyntaxException e) {
+            e.printStackTrace();
+        }
+        connection = mqtt.blockingConnection();
+        try {
+            connection.connect();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
     }
 
+    @Deprecated
     public void addLine(Object[] elements){
+        //TODO: Remove
         for(Object element:elements)
             buffer += element + ",";
         buffer += "\n";
     }
 
+    @Deprecated
     public void writeOut(){
+        //TODO: Remove
         try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(file, true))){
             bufferedWriter.append(buffer);
             bufferedWriter.flush();
@@ -75,4 +101,15 @@ public class EvaluationLogger {
         }
     }
 
+    public void logMQTT(String topic, Object[] elements){
+        String message = "";
+        for(Object element:elements)
+            message += element + ",";
+        try {
+            connection.publish(topic, message.getBytes(StandardCharsets.UTF_8), QoS.AT_LEAST_ONCE, false);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
 }
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java
index 40f89ea..727cc16 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java
@@ -38,6 +38,7 @@ public class InvocableEntityResource extends AbstractResource {
     private static final String SLASH = "/";
     private static final String DATA_PROCESSOR_PREFIX = "sepa";
     private static final String DATA_SINK_PREFIX = "sec";
+    private static int nrRuns = 1;
 
     @POST
     @Path("/register")
@@ -82,12 +83,11 @@ public class InvocableEntityResource extends AbstractResource {
         //TODO: Remove Logger after debugging
         InvocableStreamPipesEntity graph = RunningInvocableInstances.INSTANCE.get(runningInstanceId);
         EvaluationLogger logger = EvaluationLogger.getInstance();
-        Object[] line = {System.currentTimeMillis() ,"Element detached", graph.getAppId(), runningInstanceId};
-        logger.addLine(line);
+        Object[] line = {System.currentTimeMillis() ,"Element detached"};
+        logger.logMQTT("Offloading", line);
         Response resp = PipelineElementManager.getInstance().detach(graph, runningInstanceId);
         RunningInvocableInstances.INSTANCE.remove(runningInstanceId);
 
-        logger.writeOut();
         return ok(resp);
     }
 
@@ -101,9 +101,8 @@ public class InvocableEntityResource extends AbstractResource {
                                             PipelineElementReconfigurationEntity reconfigurationEntity) {
         //TODO: Remove Logger after debugging
         EvaluationLogger logger = EvaluationLogger.getInstance();
-        Object[] line = {System.currentTimeMillis() ,"reconfiguration request received", ((FreeTextStaticProperty) reconfigurationEntity.getReconfiguredStaticProperties().get(0)).getValue()};
-        logger.addLine(line);
-        logger.writeOut();
+        Object[] line = {System.currentTimeMillis() ,"reconfiguration request received", nrRuns++,((FreeTextStaticProperty) reconfigurationEntity.getReconfiguredStaticProperties().get(0)).getValue()};
+        logger.logMQTT("Reconfiguration", line);
         InvocableStreamPipesEntity graph = RunningInvocableInstances.INSTANCE.get(runningInstanceId);
         return ok(PipelineElementManager.getInstance().reconfigure(graph, reconfigurationEntity));
     }
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java
index 39aeb51..197dfe2 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java
@@ -71,8 +71,7 @@ public class OffloadingPolicyManager {
             // account
             //TODO: Remove Logger after debugging
             Object[] line = {System.currentTimeMillis() ,"offloading triggered", violatedPolicies.get(0).getOffloadingPolicy().getClass().getSimpleName()};
-            logger.addLine(line);
-            logger.writeOut();
+            logger.logMQTT("Offloading", line);
             triggerOffloading(violatedPolicies.get(0));
         }
         //Blacklist of entities is cleared when no policies were violated.
@@ -81,8 +80,8 @@ public class OffloadingPolicyManager {
 
     private void triggerOffloading(OffloadingStrategy strategy){
         InvocableStreamPipesEntity offloadEntity = strategy.getSelectionStrategy().select(this.unsuccessfullyTriedEntities);
-        Object[] line = {System.currentTimeMillis() ,"entity to offload selected", strategy.getOffloadingPolicy().getClass().getSimpleName(), offloadEntity.getAppId()};
-        logger.addLine(line);
+        Object[] line = {System.currentTimeMillis() ,"entity to offload selected"};
+        logger.logMQTT("Offloading", line);
         if(offloadEntity != null){
             Response resp = PipelineElementManager.getInstance().offload(offloadEntity);
 
@@ -90,8 +89,7 @@ public class OffloadingPolicyManager {
             String pipelineName = offloadEntity.getCorrespondingPipeline();
 
             Object[] line_done = {System.currentTimeMillis() ,"offloading done", strategy.getOffloadingPolicy().getClass().getSimpleName(), appId};
-            logger.addLine(line_done);
-            logger.writeOut();
+            logger.logMQTT("Offloading", line_done);
 
             if(resp.isSuccess()){
                 LOG.info("Successfully offloaded: {} of pipeline: {}", appId, pipelineName);
diff --git a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/EdgeExtensionsGenericPerformanceTest.java b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/EdgeExtensionsGenericPerformanceTest.java
index 23c721e..9c38672 100644
--- a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/EdgeExtensionsGenericPerformanceTest.java
+++ b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/EdgeExtensionsGenericPerformanceTest.java
@@ -26,18 +26,17 @@ public class EdgeExtensionsGenericPerformanceTest {
     public static void main(String ... args){
 
         int numberOfRuns = Integer.parseInt(System.getenv("TEST_RUNS"));
+        String testType = System.getenv("TEST_TYPE");
 
         EvaluationLogger logger = EvaluationLogger.getInstance();
 
         Test test = TestFactory.getFromEnv();
 
         for(int i = 1; i <= numberOfRuns; i++){
-            Object[] line = {System.currentTimeMillis(), String.format("Run %d started", i)};
-            logger.addLine(line);
+            Object[] line = {System.currentTimeMillis(), String.format("Run %d started", i), i};
+            logger.logMQTT(testType, line);
             test.execute();
         }
-        logger.writeOut();
-
     }
 
 }
diff --git a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/TestFactory.java b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/TestFactory.java
index 3389df2..c5f2a8c 100644
--- a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/TestFactory.java
+++ b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/TestFactory.java
@@ -17,6 +17,7 @@
  */
 package org.apache.streampipes.performance;
 
+import org.apache.streampipes.logging.evaluation.EvaluationLogger;
 import org.apache.streampipes.performance.performancetest.GenericTest;
 import org.apache.streampipes.performance.performancetest.Test;
 
@@ -24,14 +25,21 @@ public class TestFactory {
 
 
     public static Test getFromEnv(){
+        EvaluationLogger logger = EvaluationLogger.getInstance();
         switch (System.getenv("TEST_TYPE")){
             case "Deployment":
+                Object[] header_deployment = {"timestampInMillis", "event", "numberOfRuns", "durationInNanos", "durationInSecs"};
+                logger.logMQTT("Deployment", header_deployment);
                 return getDeploymentTest();
             case "Latency":
                 return getLatencyTest();
             case "Migration":
+                Object[] header_migration = {"timestampInMillis", "event", "numberOfRuns", "durationInNanos", "durationInSecs"};
+                logger.logMQTT("Migration", header_migration);
                 return getMigrationTest();
             case "Reconfiguration":
+                Object[] header_reconfigure = {"timestampInMillis", "event", "numberOfRuns", "reconfigurationValue"};
+                logger.logMQTT("Reconfiguration", header_reconfigure);
                 return getReconfigurationTest();
             default:
                 throw new RuntimeException("No test configuration found.");
diff --git a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/performancetest/GenericTest.java b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/performancetest/GenericTest.java
index 173a206..5b11443 100644
--- a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/performancetest/GenericTest.java
+++ b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/performancetest/GenericTest.java
@@ -62,13 +62,17 @@ public class GenericTest implements Test{
     @Override
     public void execute() {
 
+        String testType = System.getenv("TEST_TYPE");
+
         //Start Pipeline
         if (!pipeline.isRunning()) {
             long beforeStart = System.nanoTime();
             PipelineOperationStatus startMessage = client.pipelines().start(pipeline);
             long deploymentDuration = System.nanoTime() - beforeStart;
-            Object[] line = {System.currentTimeMillis() ,"Deployment duration", deploymentDuration};
-            evalLogger.addLine(line);
+            if(testType.equals("Deployment")){
+                Object[] line = {System.currentTimeMillis() ,"Deployment duration", deploymentDuration};
+                evalLogger.logMQTT(testType, line);
+            }
             if (startMessage.isSuccess()) {
                 System.out.println(startMessage.getTitle());
                 pipeline.setRunning(true);
@@ -88,8 +92,10 @@ public class GenericTest implements Test{
             long beforeMigration = System.nanoTime();
             PipelineOperationStatus migrationMessage = client.pipelines().migrate(pipeline);
             long migrationDuration = System.nanoTime() - beforeMigration;
-            Object[] line = {System.currentTimeMillis(), "Migration duration", migrationDuration};
-            evalLogger.addLine(line);
+            if(testType.equals("Migration")){
+                Object[] line = {System.currentTimeMillis(), "Migration duration", migrationDuration};
+                evalLogger.logMQTT(testType, line);
+            }
             if (migrationMessage.isSuccess()) {
                 System.out.println(migrationMessage.getTitle());
             }