You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2023/01/20 13:41:07 UTC

[streampipes] 01/01: Support output data streams in functions (#1133)

This is an automated email from the ASF dual-hosted git repository.

riemer pushed a commit to branch SP-1133
in repository https://gitbox.apache.org/repos/asf/streampipes.git

commit 96ee8cce00352ca6de1cf0662a4303c5a1087704
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Fri Jan 20 14:40:43 2023 +0100

    Support output data streams in functions (#1133)
---
 .../apache/streampipes/client/api/AdminApi.java    | 12 ++++
 streampipes-config/pom.xml                         |  5 ++
 .../streampipes/config/backend/BackendConfig.java  |  3 +-
 ...Settings.java => DefaultMessagingSettings.java} | 83 ++--------------------
 .../connect/management/util/GroundingUtils.java    |  2 +-
 ...sFunctionDeclarer.java => IFunctionConfig.java} | 11 ++-
 .../api/declarer/IStreamPipesFunctionDeclarer.java |  4 +-
 .../connect/adapter/AdapterPipelineGenerator.java  |  2 +-
 .../adapter/util/TransportFormatGenerator.java     |  2 +-
 .../management/init/DeclarersSingleton.java        |  2 +-
 .../model/config}/MessagingSettings.java           | 32 +--------
 .../streampipes/model/config}/SpDataFormat.java    |  2 +-
 .../streampipes/model/config}/SpProtocol.java      |  2 +-
 .../manager/matching/DataSetGroundingSelector.java |  2 +-
 .../manager/matching/FormatSelector.java           |  2 +-
 .../manager/matching/ProtocolSelector.java         |  2 +-
 .../streampipes/rest/impl/admin/ConsulConfig.java  |  2 +-
 .../function/StreamPipesFunctionHandler.java       | 34 ++++++++-
 .../standalone/function/FunctionConfig.java        | 30 ++++++--
 .../standalone/function/FunctionConfigBuilder.java | 25 +++++--
 .../standalone/function/FunctionContext.java       |  9 +++
 .../function/FunctionContextGenerator.java         | 10 ++-
 .../standalone/function/StreamPipesFunction.java   | 55 ++++++++++----
 23 files changed, 176 insertions(+), 157 deletions(-)

diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java
index edffe416c..1fbc790a4 100644
--- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java
+++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java
@@ -19,6 +19,7 @@ package org.apache.streampipes.client.api;
 
 import org.apache.streampipes.client.model.StreamPipesClientConfig;
 import org.apache.streampipes.client.util.StreamPipesApiPath;
+import org.apache.streampipes.model.config.MessagingSettings;
 import org.apache.streampipes.model.connect.adapter.AdapterDescription;
 import org.apache.streampipes.model.function.FunctionDefinition;
 import org.apache.streampipes.model.message.SuccessMessage;
@@ -43,6 +44,17 @@ public class AdminApi extends AbstractClientApi {
     delete(getDeleteFunctionPath(functionId), SuccessMessage.class);
   }
 
+  public MessagingSettings getMessagingSettings() {
+    return getSingle(getMessagingSettingsPath(), MessagingSettings.class);
+  }
+
+  private StreamPipesApiPath getMessagingSettingsPath() {
+    return StreamPipesApiPath
+        .fromBaseApiPath()
+        .addToPath("consul")
+        .addToPath("messaging");
+  }
+
   private StreamPipesApiPath getConnectPath() {
     return StreamPipesApiPath
         .fromBaseApiPath()
diff --git a/streampipes-config/pom.xml b/streampipes-config/pom.xml
index c8e261525..87b25d6bc 100644
--- a/streampipes-config/pom.xml
+++ b/streampipes-config/pom.xml
@@ -30,6 +30,11 @@
 
     <dependencies>
         <!-- StreamPipes dependencies -->
+        <dependency>
+            <groupId>org.apache.streampipes</groupId>
+            <artifactId>streampipes-model</artifactId>
+            <version>0.91.0-SNAPSHOT</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.streampipes</groupId>
             <artifactId>streampipes-serializers-json</artifactId>
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java
index b077f813e..1c02d9883 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java
+++ b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.commons.random.TokenGenerator;
 import org.apache.streampipes.config.backend.model.EmailConfig;
 import org.apache.streampipes.config.backend.model.GeneralConfig;
 import org.apache.streampipes.config.backend.model.LocalAuthConfig;
+import org.apache.streampipes.model.config.MessagingSettings;
 import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
 import org.apache.streampipes.svcdiscovery.api.SpConfig;
 
@@ -76,7 +77,7 @@ public enum BackendConfig {
     config.register(BackendConfigKeys.INFLUX_HOST, "influxdb", "The host of the influx data base");
     config.register(BackendConfigKeys.INFLUX_PORT, 8086, "The hist of the influx data base");
     config.register(BackendConfigKeys.INFLUX_DATA_BASE, "sp", "The influx data base name");
-    config.registerObject(BackendConfigKeys.MESSAGING_SETTINGS, MessagingSettings.fromDefault(),
+    config.registerObject(BackendConfigKeys.MESSAGING_SETTINGS, DefaultMessagingSettings.make(),
         "Default Messaging Settings");
 
     config.registerObject(BackendConfigKeys.LOCAL_AUTH_CONFIG, LocalAuthConfig.fromDefaults(getJwtSecret()),
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/DefaultMessagingSettings.java
similarity index 52%
copy from streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java
copy to streampipes-config/src/main/java/org/apache/streampipes/config/backend/DefaultMessagingSettings.java
index fe7710e88..4f65a7289 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java
+++ b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/DefaultMessagingSettings.java
@@ -15,22 +15,19 @@
  * limitations under the License.
  *
  */
+
 package org.apache.streampipes.config.backend;
 
+import org.apache.streampipes.model.config.MessagingSettings;
+import org.apache.streampipes.model.config.SpDataFormat;
+import org.apache.streampipes.model.config.SpProtocol;
+
 import java.util.Arrays;
 import java.util.List;
 
-public class MessagingSettings {
-
-  private Integer batchSize;
-  private Integer messageMaxBytes;
-  private Integer lingerMs;
-  private Integer acks;
+public class DefaultMessagingSettings {
 
-  private List<SpDataFormat> prioritizedFormats;
-  private List<SpProtocol> prioritizedProtocols;
-
-  public static MessagingSettings fromDefault() {
+  public static MessagingSettings make() {
     List<SpProtocol> protocolList;
     if (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL) != null) {
       switch (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL).toLowerCase()) {
@@ -58,70 +55,4 @@ public class MessagingSettings {
         Arrays.asList(SpDataFormat.JSON, SpDataFormat.CBOR, SpDataFormat.FST, SpDataFormat.SMILE),
         protocolList);
   }
-
-  public MessagingSettings(Integer batchSize,
-                           Integer messageMaxBytes,
-                           Integer lingerMs,
-                           Integer acks,
-                           List<SpDataFormat> prioritizedFormats,
-                           List<SpProtocol> prioritizedProtocols) {
-    this.batchSize = batchSize;
-    this.messageMaxBytes = messageMaxBytes;
-    this.lingerMs = lingerMs;
-    this.acks = acks;
-    this.prioritizedFormats = prioritizedFormats;
-    this.prioritizedProtocols = prioritizedProtocols;
-  }
-
-  public MessagingSettings() {
-
-  }
-
-  public Integer getBatchSize() {
-    return batchSize;
-  }
-
-  public void setBatchSize(Integer batchSize) {
-    this.batchSize = batchSize;
-  }
-
-  public Integer getMessageMaxBytes() {
-    return messageMaxBytes;
-  }
-
-  public void setMessageMaxBytes(Integer messageMaxBytes) {
-    this.messageMaxBytes = messageMaxBytes;
-  }
-
-  public Integer getLingerMs() {
-    return lingerMs;
-  }
-
-  public void setLingerMs(Integer lingerMs) {
-    this.lingerMs = lingerMs;
-  }
-
-  public Integer getAcks() {
-    return acks;
-  }
-
-  public void setAcks(Integer acks) {
-    this.acks = acks;
-  }
-
-  public List<SpDataFormat> getPrioritizedFormats() {
-    return prioritizedFormats;
-  }
-
-  public void setPrioritizedFormats(List<SpDataFormat> prioritizedFormats) {
-    this.prioritizedFormats = prioritizedFormats;
-  }
-
-  public List<SpProtocol> getPrioritizedProtocols() {
-    return prioritizedProtocols;
-  }
-
-  public void setPrioritizedProtocols(List<SpProtocol> prioritizedProtocols) {
-    this.prioritizedProtocols = prioritizedProtocols;
-  }
 }
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java
index b62cb9351..b820956cd 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java
@@ -19,8 +19,8 @@
 package org.apache.streampipes.connect.management.util;
 
 import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpProtocol;
 import org.apache.streampipes.extensions.management.connect.adapter.util.TransportFormatGenerator;
+import org.apache.streampipes.model.config.SpProtocol;
 import org.apache.streampipes.model.grounding.EventGrounding;
 import org.apache.streampipes.model.grounding.JmsTransportProtocol;
 import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IFunctionConfig.java
similarity index 80%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
copy to streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IFunctionConfig.java
index 9f35a7bfa..16baf52ca 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
+++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IFunctionConfig.java
@@ -18,18 +18,17 @@
 
 package org.apache.streampipes.extensions.api.declarer;
 
+import org.apache.streampipes.model.SpDataStream;
 import org.apache.streampipes.model.function.FunctionId;
 
-import java.util.List;
+import java.util.Map;
 
-public interface IStreamPipesFunctionDeclarer {
+public interface IFunctionConfig {
 
   FunctionId getFunctionId();
 
-  List<String> requiredStreamIds();
+  Map<String, SpDataStream> getOutputDataStreams();
 
-  void invokeRuntime(String serviceGroup);
-
-  void discardRuntime();
+  void addOutputDataStream(String streamId, SpDataStream stream);
 
 }
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
index 9f35a7bfa..cdf003637 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
+++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
@@ -18,13 +18,11 @@
 
 package org.apache.streampipes.extensions.api.declarer;
 
-import org.apache.streampipes.model.function.FunctionId;
-
 import java.util.List;
 
 public interface IStreamPipesFunctionDeclarer {
 
-  FunctionId getFunctionId();
+  IFunctionConfig getFunctionConfig();
 
   List<String> requiredStreamIds();
 
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java
index 60d409a5e..c186688a4 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java
@@ -19,7 +19,6 @@
 package org.apache.streampipes.extensions.management.connect.adapter;
 
 import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpProtocol;
 import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement;
 import org.apache.streampipes.extensions.management.connect.adapter.model.pipeline.AdapterPipeline;
 import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.AddTimestampPipelineElement;
@@ -34,6 +33,7 @@ import org.apache.streampipes.extensions.management.connect.adapter.preprocessin
 import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.TransformStreamAdapterElement;
 import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.TransformValueAdapterPipelineElement;
 import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.transform.stream.DuplicateFilterPipelineElement;
+import org.apache.streampipes.model.config.SpProtocol;
 import org.apache.streampipes.model.connect.adapter.AdapterDescription;
 import org.apache.streampipes.model.connect.rules.DebugSinkRuleDescription;
 import org.apache.streampipes.model.connect.rules.TransformationRuleDescription;
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatGenerator.java
index 84e57d1b8..d7e0c1390 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatGenerator.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatGenerator.java
@@ -18,7 +18,7 @@
 package org.apache.streampipes.extensions.management.connect.adapter.util;
 
 import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpDataFormat;
+import org.apache.streampipes.model.config.SpDataFormat;
 import org.apache.streampipes.model.grounding.TransportFormat;
 import org.apache.streampipes.sdk.helpers.SupportedFormats;
 import org.apache.streampipes.vocabulary.MessageFormat;
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java
index a1e04a735..32ad490dc 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java
@@ -111,7 +111,7 @@ public class DeclarersSingleton {
     this.registerDataFormats(serviceDef.getDataFormatFactories());
     this.allAdapters = serviceDef.getSpecificAdapters();
     this.allProtocols = serviceDef.getAdapterProtocols();
-    serviceDef.getFunctions().forEach(f -> this.functions.put(f.getFunctionId().getId(), f));
+    serviceDef.getFunctions().forEach(f -> this.functions.put(f.getFunctionConfig().getFunctionId().getId(), f));
 
   }
 
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java b/streampipes-model/src/main/java/org/apache/streampipes/model/config/MessagingSettings.java
similarity index 66%
rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java
rename to streampipes-model/src/main/java/org/apache/streampipes/model/config/MessagingSettings.java
index fe7710e88..277aa3bf3 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/config/MessagingSettings.java
@@ -15,9 +15,8 @@
  * limitations under the License.
  *
  */
-package org.apache.streampipes.config.backend;
+package org.apache.streampipes.model.config;
 
-import java.util.Arrays;
 import java.util.List;
 
 public class MessagingSettings {
@@ -30,34 +29,7 @@ public class MessagingSettings {
   private List<SpDataFormat> prioritizedFormats;
   private List<SpProtocol> prioritizedProtocols;
 
-  public static MessagingSettings fromDefault() {
-    List<SpProtocol> protocolList;
-    if (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL) != null) {
-      switch (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL).toLowerCase()) {
-        case "mqtt":
-          protocolList = Arrays.asList(SpProtocol.MQTT, SpProtocol.KAFKA, SpProtocol.JMS, SpProtocol.NATS);
-          break;
-        case "kafka":
-          protocolList = Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS);
-          break;
-        case "jms":
-          protocolList = Arrays.asList(SpProtocol.JMS, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.NATS);
-          break;
-        case "nats":
-          protocolList = Arrays.asList(SpProtocol.NATS, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS);
-          break;
-        default:
-          protocolList = Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS);
-      }
-    } else {
-      protocolList = Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS);
-    }
-
-    return new MessagingSettings(
-        1638400, 5000012, 20, 2,
-        Arrays.asList(SpDataFormat.JSON, SpDataFormat.CBOR, SpDataFormat.FST, SpDataFormat.SMILE),
-        protocolList);
-  }
+
 
   public MessagingSettings(Integer batchSize,
                            Integer messageMaxBytes,
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpDataFormat.java b/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpDataFormat.java
similarity index 96%
rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpDataFormat.java
rename to streampipes-model/src/main/java/org/apache/streampipes/model/config/SpDataFormat.java
index 411f457cc..c409eeaab 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpDataFormat.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpDataFormat.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  *
  */
-package org.apache.streampipes.config.backend;
+package org.apache.streampipes.model.config;
 
 
 import org.apache.streampipes.vocabulary.MessageFormat;
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java b/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpProtocol.java
similarity index 96%
rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java
rename to streampipes-model/src/main/java/org/apache/streampipes/model/config/SpProtocol.java
index b69ce31fb..bd6f2a4e9 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpProtocol.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  *
  */
-package org.apache.streampipes.config.backend;
+package org.apache.streampipes.model.config;
 
 public enum SpProtocol {
 
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/DataSetGroundingSelector.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/DataSetGroundingSelector.java
index 75e6f5383..1a419b747 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/DataSetGroundingSelector.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/DataSetGroundingSelector.java
@@ -18,9 +18,9 @@
 package org.apache.streampipes.manager.matching;
 
 import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpProtocol;
 import org.apache.streampipes.manager.util.TopicGenerator;
 import org.apache.streampipes.model.SpDataSet;
+import org.apache.streampipes.model.config.SpProtocol;
 import org.apache.streampipes.model.grounding.EventGrounding;
 import org.apache.streampipes.model.grounding.JmsTransportProtocol;
 import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java
index ba9b3d345..128098f7c 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java
@@ -19,10 +19,10 @@
 package org.apache.streampipes.manager.matching;
 
 import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpDataFormat;
 import org.apache.streampipes.model.SpDataStream;
 import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
 import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.model.config.SpDataFormat;
 import org.apache.streampipes.model.grounding.TransportFormat;
 import org.apache.streampipes.vocabulary.MessageFormat;
 
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java
index 48e211cb8..9631a4cad 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java
@@ -19,11 +19,11 @@
 package org.apache.streampipes.manager.matching;
 
 import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpProtocol;
 import org.apache.streampipes.manager.util.TopicGenerator;
 import org.apache.streampipes.model.SpDataStream;
 import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
 import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.model.config.SpProtocol;
 import org.apache.streampipes.model.grounding.JmsTransportProtocol;
 import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
 import org.apache.streampipes.model.grounding.MqttTransportProtocol;
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java
index 87b3d68f4..17bdb22a8 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java
@@ -20,7 +20,7 @@ package org.apache.streampipes.rest.impl.admin;
 
 
 import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.MessagingSettings;
+import org.apache.streampipes.model.config.MessagingSettings;
 import org.apache.streampipes.rest.core.base.impl.AbstractRestResource;
 import org.apache.streampipes.rest.security.AuthConstants;
 import org.apache.streampipes.rest.shared.annotation.JacksonSerialized;
diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/StreamPipesFunctionHandler.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/StreamPipesFunctionHandler.java
index 46d8f135f..a9ec6cc25 100644
--- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/StreamPipesFunctionHandler.java
+++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/StreamPipesFunctionHandler.java
@@ -18,8 +18,11 @@
 
 package org.apache.streampipes.service.extensions.function;
 
+import org.apache.streampipes.extensions.api.declarer.DataStreamDeclarer;
 import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer;
+import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver;
 import org.apache.streampipes.extensions.management.init.DeclarersSingleton;
+import org.apache.streampipes.model.SpDataStream;
 import org.apache.streampipes.model.function.FunctionDefinition;
 
 import java.util.Collection;
@@ -44,15 +47,40 @@ public enum StreamPipesFunctionHandler {
 
   public void initializeFunctions(Collection<IStreamPipesFunctionDeclarer> functions,
                                   String serviceGroup) {
+    registerDataStreams(functions);
     functions.forEach(function -> {
       function.invokeRuntime(serviceGroup);
-      runningInstances.put(function.getFunctionId().getId(), function);
+      runningInstances.put(function.getFunctionConfig().getFunctionId().getId(), function);
     });
     if (!(runningInstances.isEmpty())) {
       new Thread(new FunctionRegistrationHandler(getFunctionDefinitions())).start();
     }
   }
 
+  public void registerDataStreams(Collection<IStreamPipesFunctionDeclarer> functions) {
+    var client = new StreamPipesClientResolver().makeStreamPipesClientInstance();
+    functions.forEach(function -> {
+      function.getFunctionConfig().getOutputDataStreams().values().forEach(ds -> {
+        DeclarersSingleton.getInstance().add(new DataStreamDeclarer() {
+          @Override
+          public SpDataStream declareModel() {
+            return ds;
+          }
+
+          @Override
+          public void executeStream() {
+
+          }
+
+          @Override
+          public boolean isExecutable() {
+            return false;
+          }
+        });
+      });
+    });
+  }
+
   public void cleanupFunctions() {
     this.runningInstances.forEach((key, value) -> {
       value.discardRuntime();
@@ -64,7 +92,9 @@ public enum StreamPipesFunctionHandler {
     return this.runningInstances
         .values()
         .stream()
-        .map(function -> new FunctionDefinition(function.getFunctionId(), function.requiredStreamIds()))
+        .map(function -> new FunctionDefinition(
+            function.getFunctionConfig().getFunctionId(),
+            function.requiredStreamIds()))
         .collect(Collectors.toList());
   }
 
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfig.java
similarity index 50%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
copy to streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfig.java
index 9f35a7bfa..3e7448c07 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfig.java
@@ -16,20 +16,36 @@
  *
  */
 
-package org.apache.streampipes.extensions.api.declarer;
+package org.apache.streampipes.wrapper.standalone.function;
 
+import org.apache.streampipes.extensions.api.declarer.IFunctionConfig;
+import org.apache.streampipes.model.SpDataStream;
 import org.apache.streampipes.model.function.FunctionId;
 
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
 
-public interface IStreamPipesFunctionDeclarer {
+public class FunctionConfig implements IFunctionConfig {
 
-  FunctionId getFunctionId();
+  private final FunctionId functionId;
+  private final Map<String, SpDataStream> outputDataStreams;
 
-  List<String> requiredStreamIds();
+  public FunctionConfig(FunctionId functionId) {
+    this.functionId = functionId;
+    this.outputDataStreams = new HashMap<>();
+  }
 
-  void invokeRuntime(String serviceGroup);
+  public FunctionId getFunctionId() {
+    return functionId;
+  }
 
-  void discardRuntime();
+  public Map<String, SpDataStream> getOutputDataStreams() {
+    return outputDataStreams;
+  }
+
+  public void addOutputDataStream(String streamId,
+                                  SpDataStream stream) {
+    this.outputDataStreams.put(streamId, stream);
+  }
 
 }
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfigBuilder.java
similarity index 56%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
copy to streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfigBuilder.java
index 9f35a7bfa..0d2c8e246 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfigBuilder.java
@@ -16,20 +16,31 @@
  *
  */
 
-package org.apache.streampipes.extensions.api.declarer;
+package org.apache.streampipes.wrapper.standalone.function;
 
+import org.apache.streampipes.model.SpDataStream;
 import org.apache.streampipes.model.function.FunctionId;
 
-import java.util.List;
+public class FunctionConfigBuilder {
 
-public interface IStreamPipesFunctionDeclarer {
+  private final FunctionConfig functionConfig;
 
-  FunctionId getFunctionId();
+  public static FunctionConfigBuilder create(FunctionId functionId) {
+    return new FunctionConfigBuilder(functionId);
+  }
 
-  List<String> requiredStreamIds();
+  private FunctionConfigBuilder(FunctionId functionId) {
+    this.functionConfig = new FunctionConfig(functionId);
+  }
 
-  void invokeRuntime(String serviceGroup);
+  public FunctionConfigBuilder withOutputStream(SpDataStream stream) {
+    this.functionConfig.addOutputDataStream(stream.getAppId(), stream);
+    return this;
+  }
+
+  public FunctionConfig build() {
+    return functionConfig;
+  }
 
-  void discardRuntime();
 
 }
diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java
index dc8370605..5d015c64b 100644
--- a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.extensions.management.monitoring.SpMonitoringManag
 import org.apache.streampipes.model.SpDataStream;
 import org.apache.streampipes.model.monitoring.SpLogEntry;
 import org.apache.streampipes.model.schema.EventSchema;
+import org.apache.streampipes.wrapper.routing.SpOutputCollector;
 
 import java.util.Collection;
 import java.util.HashMap;
@@ -33,10 +34,13 @@ import java.util.Map;
 public class FunctionContext {
 
   private final Map<String, SpDataStream> streams;
+
   private StreamPipesClient client;
   private String functionId;
   private ConfigExtractor config;
 
+  private Map<String, SpOutputCollector> outputCollectors;
+
   public FunctionContext() {
     this.streams = new HashMap<>();
   }
@@ -44,11 +48,13 @@ public class FunctionContext {
   public FunctionContext(String functionId,
                          String serviceGroup,
                          List<SpDataStream> streams,
+                         Map<String, SpOutputCollector> outputCollectors,
                          StreamPipesClient client) {
     this();
     streams.forEach(stream -> this.streams.put(stream.getElementId(), stream));
     this.config = ConfigExtractor.from(serviceGroup);
     this.functionId = functionId;
+    this.outputCollectors = outputCollectors;
     this.client = client;
   }
 
@@ -76,4 +82,7 @@ public class FunctionContext {
     SpMonitoringManager.INSTANCE.addErrorMessage(functionId, logEntry);
   }
 
+  public Map<String, SpOutputCollector> getOutputCollectors() {
+    return outputCollectors;
+  }
 }
diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java
index 483f712c1..c3465b413 100644
--- a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java
@@ -21,8 +21,10 @@ package org.apache.streampipes.wrapper.standalone.function;
 import org.apache.streampipes.client.StreamPipesClient;
 import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver;
 import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.wrapper.routing.SpOutputCollector;
 
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 public class FunctionContextGenerator {
@@ -31,19 +33,23 @@ public class FunctionContextGenerator {
   private final String functionId;
   private final String serviceGroup;
 
+  private final Map<String, SpOutputCollector> outputCollectors;
+
   public FunctionContextGenerator(String functionId,
                                   String serviceGroup,
-                                  List<String> streamIds) {
+                                  List<String> streamIds,
+                                  Map<String, SpOutputCollector> outputCollectors) {
     this.streamIds = streamIds;
     this.serviceGroup = serviceGroup;
     this.functionId = functionId;
+    this.outputCollectors = outputCollectors;
   }
 
   public FunctionContext generate() {
     var client = makeClient();
     var streams = receiveStreams(client);
 
-    return new FunctionContext(functionId, serviceGroup, streams, client);
+    return new FunctionContext(functionId, serviceGroup, streams, outputCollectors, client);
   }
 
   private StreamPipesClient makeClient() {
diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java
index 547dc2dfe..8f79526e2 100644
--- a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.wrapper.standalone.function;
 
 import org.apache.streampipes.commons.constants.Envs;
 import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.extensions.api.declarer.IFunctionConfig;
 import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer;
 import org.apache.streampipes.extensions.management.monitoring.SpMonitoringManager;
 import org.apache.streampipes.extensions.management.util.GroundingDebugUtils;
@@ -34,6 +35,7 @@ import org.apache.streampipes.model.runtime.SourceInfo;
 import org.apache.streampipes.model.schema.EventSchema;
 import org.apache.streampipes.wrapper.routing.RawDataProcessor;
 import org.apache.streampipes.wrapper.routing.SpInputCollector;
+import org.apache.streampipes.wrapper.routing.SpOutputCollector;
 import org.apache.streampipes.wrapper.standalone.manager.ProtocolManager;
 
 import org.slf4j.Logger;
@@ -52,16 +54,23 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
   private final Map<String, SchemaInfo> schemaInfoMapper;
   private Map<String, SpInputCollector> inputCollectors;
 
+  private Map<String, SpOutputCollector> outputCollectors;
+
   public StreamPipesFunction() {
     this.sourceInfoMapper = new HashMap<>();
     this.schemaInfoMapper = new HashMap<>();
+    this.outputCollectors = new HashMap<>();
   }
 
   @Override
   public void invokeRuntime(String serviceGroup) {
+    var functionId = this.getFunctionConfig().getFunctionId();
+    this.outputCollectors = this.getOutputCollectors();
+    this.outputCollectors.forEach((key, value) -> value.connect());
+
     FunctionContext context = new FunctionContextGenerator(
-        this.getFunctionId().getId(), serviceGroup, this.requiredStreamIds()
-    ).generate();
+        functionId.getId(), serviceGroup, this.requiredStreamIds(), this.outputCollectors)
+        .generate();
 
     // Creates a source info for each incoming SpDataStream
     // The index is used to create the selector prefix for the SourceInfo
@@ -76,15 +85,18 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
         });
 
     this.inputCollectors = getInputCollectors(context.getStreams());
-    LOG.info("Invoking function {}:{}", this.getFunctionId().getId(), this.getFunctionId().getVersion());
+
+    LOG.info("Invoking function {}:{}", functionId.getId(), functionId.getVersion());
     onServiceStarted(context);
     registerConsumers();
   }
 
   public void discardRuntime() {
-    LOG.info("Discarding function {}:{}", this.getFunctionId().getId(), this.getFunctionId().getVersion());
+    var functionId = this.getFunctionConfig().getFunctionId();
+    LOG.info("Discarding function {}:{}", functionId.getId(), functionId.getVersion());
     onServiceStopped();
     unregisterConsumers();
+    this.outputCollectors.forEach((key, value) -> value.disconnect());
   }
 
   @Override
@@ -107,19 +119,34 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
   }
 
   private void increaseCounter(String sourceInfo) {
+    var functionId = this.getFunctionConfig().getFunctionId();
     SpMonitoringManager.INSTANCE.increaseInCounter(
-        this.getFunctionId().getId(),
+        functionId.getId(),
         sourceInfo,
         System.currentTimeMillis()
     );
   }
 
   private void addError(RuntimeException e) {
+    var functionId = this.getFunctionConfig().getFunctionId();
     SpMonitoringManager.INSTANCE.addErrorMessage(
-        this.getFunctionId().getId(),
+        functionId.getId(),
         SpLogEntry.from(System.currentTimeMillis(), StreamPipesErrorMessage.from(e)));
   }
 
+  private Map<String, SpOutputCollector> getOutputCollectors() {
+    this.getFunctionConfig().getOutputDataStreams().forEach((key, value) -> {
+      this.outputCollectors.put(
+          key,
+          ProtocolManager.makeOutputCollector(
+              value.getEventGrounding().getTransportProtocol(),
+              value.getEventGrounding().getTransportFormats().get(0),
+              key));
+    });
+
+    return this.outputCollectors;
+  }
+
   private Map<String, SpInputCollector> getInputCollectors(Collection<SpDataStream> streams) throws SpRuntimeException {
     Map<String, SpInputCollector> inputCollectors = new HashMap<>();
     for (SpDataStream is : streams) {
@@ -146,13 +173,6 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
     });
   }
 
-  public abstract void onServiceStarted(FunctionContext context);
-
-  public abstract void onEvent(Event event,
-                               String streamId);
-
-  public abstract void onServiceStopped();
-
   private SourceInfo createSourceInfo(SpDataStream stream, int streamIndex) {
     return new SourceInfo(
         stream.getElementId(),
@@ -164,4 +184,13 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
     return new SchemaInfo(eventSchema, new ArrayList<>());
   }
 
+  public abstract IFunctionConfig getFunctionConfig();
+
+  public abstract void onServiceStarted(FunctionContext context);
+
+  public abstract void onEvent(Event event,
+                               String streamId);
+
+  public abstract void onServiceStopped();
+
 }