You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by go...@apache.org on 2021/06/22 14:39:29 UTC
[incubator-streampipes] branch edge-extensions updated: Introduced
MQTT Logging
This is an automated email from the ASF dual-hosted git repository.
gomm pushed a commit to branch edge-extensions
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/edge-extensions by this push:
new 094bb68 Introduced MQTT Logging
094bb68 is described below
commit 094bb6836d11b7723718afa27bf9dcfa2a2f8748
Author: daniel-gomm <da...@outlook.de>
AuthorDate: Tue Jun 22 16:36:35 2021 +0200
Introduced MQTT Logging
---
streampipes-logging/pom.xml | 5 +++
.../logging/evaluation/EvaluationLogger.java | 41 ++++++++++++++++++++--
.../controller/api/InvocableEntityResource.java | 11 +++---
.../offloading/OffloadingPolicyManager.java | 10 +++---
.../EdgeExtensionsGenericPerformanceTest.java | 7 ++--
.../streampipes/performance/TestFactory.java | 8 +++++
.../performance/performancetest/GenericTest.java | 14 +++++---
7 files changed, 74 insertions(+), 22 deletions(-)
diff --git a/streampipes-logging/pom.xml b/streampipes-logging/pom.xml
index d648b2f..f2bb53c 100644
--- a/streampipes-logging/pom.xml
+++ b/streampipes-logging/pom.xml
@@ -34,5 +34,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.fusesource.mqtt-client</groupId>
+ <artifactId>mqtt-client</artifactId>
+ <version>1.12</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/streampipes-logging/src/main/java/org/apache/streampipes/logging/evaluation/EvaluationLogger.java b/streampipes-logging/src/main/java/org/apache/streampipes/logging/evaluation/EvaluationLogger.java
index 501d4e0..2d2d2bc 100644
--- a/streampipes-logging/src/main/java/org/apache/streampipes/logging/evaluation/EvaluationLogger.java
+++ b/streampipes-logging/src/main/java/org/apache/streampipes/logging/evaluation/EvaluationLogger.java
@@ -17,20 +17,26 @@
*/
package org.apache.streampipes.logging.evaluation;
+import org.fusesource.mqtt.client.BlockingConnection;
+import org.fusesource.mqtt.client.MQTT;
+import org.fusesource.mqtt.client.QoS;
+
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.Date;
public class EvaluationLogger {
- private static String FILE_NAME = "evaluation-logs.csv";
-
private final File file;
private String buffer = "";
private static EvaluationLogger instance = null;
+ private final MQTT mqtt;
+ private final BlockingConnection connection;
public static EvaluationLogger getInstance(){
if(instance==null) instance = new EvaluationLogger();
@@ -39,9 +45,12 @@ public class EvaluationLogger {
private EvaluationLogger(){
String filepath = System.getenv("SP_LOGGING_FILE_PATH");
+ String logging_host = System.getenv("SP_LOGGING_MQTT_HOST");
+ int logging_port = Integer.parseInt(System.getenv("SP_LOGGING_MQTT_PORT"));
if(filepath==null) throw new RuntimeException("No Logging Location provided.");
if(!filepath.endsWith("/")) filepath = filepath + "/";
String timePrefix = new SimpleDateFormat("yyyy-MM-dd-HHmmss").format(new Date());
+ String FILE_NAME = "evaluation-logs.csv";
String fileWithPath = new StringBuilder()
.append(filepath)
.append(timePrefix)
@@ -57,15 +66,32 @@ public class EvaluationLogger {
} catch (IOException e) {
e.printStackTrace();
}
+
+ mqtt = new MQTT();
+ try {
+ mqtt.setHost(logging_host, logging_port);
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ }
+ connection = mqtt.blockingConnection();
+ try {
+ connection.connect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
+ @Deprecated
public void addLine(Object[] elements){
+ //TODO: Remove
for(Object element:elements)
buffer += element + ",";
buffer += "\n";
}
+ @Deprecated
public void writeOut(){
+ //TODO: Remove
try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(file, true))){
bufferedWriter.append(buffer);
bufferedWriter.flush();
@@ -75,4 +101,15 @@ public class EvaluationLogger {
}
}
+ public void logMQTT(String topic, Object[] elements){
+ String message = "";
+ for(Object element:elements)
+ message += element + ",";
+ try {
+ connection.publish(topic, message.getBytes(StandardCharsets.UTF_8), QoS.AT_LEAST_ONCE, false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java
index 40f89ea..727cc16 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java
@@ -38,6 +38,7 @@ public class InvocableEntityResource extends AbstractResource {
private static final String SLASH = "/";
private static final String DATA_PROCESSOR_PREFIX = "sepa";
private static final String DATA_SINK_PREFIX = "sec";
+ private static int nrRuns = 1;
@POST
@Path("/register")
@@ -82,12 +83,11 @@ public class InvocableEntityResource extends AbstractResource {
//TODO: Remove Logger after debugging
InvocableStreamPipesEntity graph = RunningInvocableInstances.INSTANCE.get(runningInstanceId);
EvaluationLogger logger = EvaluationLogger.getInstance();
- Object[] line = {System.currentTimeMillis() ,"Element detached", graph.getAppId(), runningInstanceId};
- logger.addLine(line);
+ Object[] line = {System.currentTimeMillis() ,"Element detached"};
+ logger.logMQTT("Offloading", line);
Response resp = PipelineElementManager.getInstance().detach(graph, runningInstanceId);
RunningInvocableInstances.INSTANCE.remove(runningInstanceId);
- logger.writeOut();
return ok(resp);
}
@@ -101,9 +101,8 @@ public class InvocableEntityResource extends AbstractResource {
PipelineElementReconfigurationEntity reconfigurationEntity) {
//TODO: Remove Logger after debugging
EvaluationLogger logger = EvaluationLogger.getInstance();
- Object[] line = {System.currentTimeMillis() ,"reconfiguration request received", ((FreeTextStaticProperty) reconfigurationEntity.getReconfiguredStaticProperties().get(0)).getValue()};
- logger.addLine(line);
- logger.writeOut();
+ Object[] line = {System.currentTimeMillis() ,"reconfiguration request received", nrRuns++,((FreeTextStaticProperty) reconfigurationEntity.getReconfiguredStaticProperties().get(0)).getValue()};
+ logger.logMQTT("Reconfiguration", line);
InvocableStreamPipesEntity graph = RunningInvocableInstances.INSTANCE.get(runningInstanceId);
return ok(PipelineElementManager.getInstance().reconfigure(graph, reconfigurationEntity));
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java
index 39aeb51..197dfe2 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java
@@ -71,8 +71,7 @@ public class OffloadingPolicyManager {
// account
//TODO: Remove Logger after debugging
Object[] line = {System.currentTimeMillis() ,"offloading triggered", violatedPolicies.get(0).getOffloadingPolicy().getClass().getSimpleName()};
- logger.addLine(line);
- logger.writeOut();
+ logger.logMQTT("Offloading", line);
triggerOffloading(violatedPolicies.get(0));
}
//Blacklist of entities is cleared when no policies were violated.
@@ -81,8 +80,8 @@ public class OffloadingPolicyManager {
private void triggerOffloading(OffloadingStrategy strategy){
InvocableStreamPipesEntity offloadEntity = strategy.getSelectionStrategy().select(this.unsuccessfullyTriedEntities);
- Object[] line = {System.currentTimeMillis() ,"entity to offload selected", strategy.getOffloadingPolicy().getClass().getSimpleName(), offloadEntity.getAppId()};
- logger.addLine(line);
+ Object[] line = {System.currentTimeMillis() ,"entity to offload selected"};
+ logger.logMQTT("Offloading", line);
if(offloadEntity != null){
Response resp = PipelineElementManager.getInstance().offload(offloadEntity);
@@ -90,8 +89,7 @@ public class OffloadingPolicyManager {
String pipelineName = offloadEntity.getCorrespondingPipeline();
Object[] line_done = {System.currentTimeMillis() ,"offloading done", strategy.getOffloadingPolicy().getClass().getSimpleName(), appId};
- logger.addLine(line_done);
- logger.writeOut();
+ logger.logMQTT("Offloading", line_done);
if(resp.isSuccess()){
LOG.info("Successfully offloaded: {} of pipeline: {}", appId, pipelineName);
diff --git a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/EdgeExtensionsGenericPerformanceTest.java b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/EdgeExtensionsGenericPerformanceTest.java
index 23c721e..9c38672 100644
--- a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/EdgeExtensionsGenericPerformanceTest.java
+++ b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/EdgeExtensionsGenericPerformanceTest.java
@@ -26,18 +26,17 @@ public class EdgeExtensionsGenericPerformanceTest {
public static void main(String ... args){
int numberOfRuns = Integer.parseInt(System.getenv("TEST_RUNS"));
+ String testType = System.getenv("TEST_TYPE");
EvaluationLogger logger = EvaluationLogger.getInstance();
Test test = TestFactory.getFromEnv();
for(int i = 1; i <= numberOfRuns; i++){
- Object[] line = {System.currentTimeMillis(), String.format("Run %d started", i)};
- logger.addLine(line);
+ Object[] line = {System.currentTimeMillis(), String.format("Run %d started", i), i};
+ logger.logMQTT(testType, line);
test.execute();
}
- logger.writeOut();
-
}
}
diff --git a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/TestFactory.java b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/TestFactory.java
index 3389df2..c5f2a8c 100644
--- a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/TestFactory.java
+++ b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/TestFactory.java
@@ -17,6 +17,7 @@
*/
package org.apache.streampipes.performance;
+import org.apache.streampipes.logging.evaluation.EvaluationLogger;
import org.apache.streampipes.performance.performancetest.GenericTest;
import org.apache.streampipes.performance.performancetest.Test;
@@ -24,14 +25,21 @@ public class TestFactory {
public static Test getFromEnv(){
+ EvaluationLogger logger = EvaluationLogger.getInstance();
switch (System.getenv("TEST_TYPE")){
case "Deployment":
+ Object[] header_deployment = {"timestampInMillis", "event", "numberOfRuns", "durationInNanos", "durationInSecs"};
+ logger.logMQTT("Deployment", header_deployment);
return getDeploymentTest();
case "Latency":
return getLatencyTest();
case "Migration":
+ Object[] header_migration = {"timestampInMillis", "event", "numberOfRuns", "durationInNanos", "durationInSecs"};
+ logger.logMQTT("Migration", header_migration);
return getMigrationTest();
case "Reconfiguration":
+ Object[] header_reconfigure = {"timestampInMillis", "event", "numberOfRuns", "reconfigurationValue"};
+ logger.logMQTT("Reconfiguration", header_reconfigure);
return getReconfigurationTest();
default:
throw new RuntimeException("No test configuration found.");
diff --git a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/performancetest/GenericTest.java b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/performancetest/GenericTest.java
index 173a206..5b11443 100644
--- a/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/performancetest/GenericTest.java
+++ b/streampipes-performance-tests/src/main/java/org/apache/streampipes/performance/performancetest/GenericTest.java
@@ -62,13 +62,17 @@ public class GenericTest implements Test{
@Override
public void execute() {
+ String testType = System.getenv("TEST_TYPE");
+
//Start Pipeline
if (!pipeline.isRunning()) {
long beforeStart = System.nanoTime();
PipelineOperationStatus startMessage = client.pipelines().start(pipeline);
long deploymentDuration = System.nanoTime() - beforeStart;
- Object[] line = {System.currentTimeMillis() ,"Deployment duration", deploymentDuration};
- evalLogger.addLine(line);
+ if(testType.equals("Deployment")){
+ Object[] line = {System.currentTimeMillis() ,"Deployment duration", deploymentDuration};
+ evalLogger.logMQTT(testType, line);
+ }
if (startMessage.isSuccess()) {
System.out.println(startMessage.getTitle());
pipeline.setRunning(true);
@@ -88,8 +92,10 @@ public class GenericTest implements Test{
long beforeMigration = System.nanoTime();
PipelineOperationStatus migrationMessage = client.pipelines().migrate(pipeline);
long migrationDuration = System.nanoTime() - beforeMigration;
- Object[] line = {System.currentTimeMillis(), "Migration duration", migrationDuration};
- evalLogger.addLine(line);
+ if(testType.equals("Migration")){
+ Object[] line = {System.currentTimeMillis(), "Migration duration", migrationDuration};
+ evalLogger.logMQTT(testType, line);
+ }
if (migrationMessage.isSuccess()) {
System.out.println(migrationMessage.getTitle());
}