You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2023/01/20 13:41:07 UTC
[streampipes] 01/01: Support output data streams in functions (#1133)
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch SP-1133
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 96ee8cce00352ca6de1cf0662a4303c5a1087704
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Fri Jan 20 14:40:43 2023 +0100
Support output data streams in functions (#1133)
---
.../apache/streampipes/client/api/AdminApi.java | 12 ++++
streampipes-config/pom.xml | 5 ++
.../streampipes/config/backend/BackendConfig.java | 3 +-
...Settings.java => DefaultMessagingSettings.java} | 83 ++--------------------
.../connect/management/util/GroundingUtils.java | 2 +-
...sFunctionDeclarer.java => IFunctionConfig.java} | 11 ++-
.../api/declarer/IStreamPipesFunctionDeclarer.java | 4 +-
.../connect/adapter/AdapterPipelineGenerator.java | 2 +-
.../adapter/util/TransportFormatGenerator.java | 2 +-
.../management/init/DeclarersSingleton.java | 2 +-
.../model/config}/MessagingSettings.java | 32 +--------
.../streampipes/model/config}/SpDataFormat.java | 2 +-
.../streampipes/model/config}/SpProtocol.java | 2 +-
.../manager/matching/DataSetGroundingSelector.java | 2 +-
.../manager/matching/FormatSelector.java | 2 +-
.../manager/matching/ProtocolSelector.java | 2 +-
.../streampipes/rest/impl/admin/ConsulConfig.java | 2 +-
.../function/StreamPipesFunctionHandler.java | 34 ++++++++-
.../standalone/function/FunctionConfig.java | 30 ++++++--
.../standalone/function/FunctionConfigBuilder.java | 25 +++++--
.../standalone/function/FunctionContext.java | 9 +++
.../function/FunctionContextGenerator.java | 10 ++-
.../standalone/function/StreamPipesFunction.java | 55 ++++++++++----
23 files changed, 176 insertions(+), 157 deletions(-)
diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java
index edffe416c..1fbc790a4 100644
--- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java
+++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java
@@ -19,6 +19,7 @@ package org.apache.streampipes.client.api;
import org.apache.streampipes.client.model.StreamPipesClientConfig;
import org.apache.streampipes.client.util.StreamPipesApiPath;
+import org.apache.streampipes.model.config.MessagingSettings;
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
import org.apache.streampipes.model.function.FunctionDefinition;
import org.apache.streampipes.model.message.SuccessMessage;
@@ -43,6 +44,17 @@ public class AdminApi extends AbstractClientApi {
delete(getDeleteFunctionPath(functionId), SuccessMessage.class);
}
+ public MessagingSettings getMessagingSettings() {
+ return getSingle(getMessagingSettingsPath(), MessagingSettings.class);
+ }
+
+ private StreamPipesApiPath getMessagingSettingsPath() {
+ return StreamPipesApiPath
+ .fromBaseApiPath()
+ .addToPath("consul")
+ .addToPath("messaging");
+ }
+
private StreamPipesApiPath getConnectPath() {
return StreamPipesApiPath
.fromBaseApiPath()
diff --git a/streampipes-config/pom.xml b/streampipes-config/pom.xml
index c8e261525..87b25d6bc 100644
--- a/streampipes-config/pom.xml
+++ b/streampipes-config/pom.xml
@@ -30,6 +30,11 @@
<dependencies>
<!-- StreamPipes dependencies -->
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-model</artifactId>
+ <version>0.91.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.streampipes</groupId>
<artifactId>streampipes-serializers-json</artifactId>
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java
index b077f813e..1c02d9883 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java
+++ b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.commons.random.TokenGenerator;
import org.apache.streampipes.config.backend.model.EmailConfig;
import org.apache.streampipes.config.backend.model.GeneralConfig;
import org.apache.streampipes.config.backend.model.LocalAuthConfig;
+import org.apache.streampipes.model.config.MessagingSettings;
import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.SpConfig;
@@ -76,7 +77,7 @@ public enum BackendConfig {
config.register(BackendConfigKeys.INFLUX_HOST, "influxdb", "The host of the influx data base");
config.register(BackendConfigKeys.INFLUX_PORT, 8086, "The hist of the influx data base");
config.register(BackendConfigKeys.INFLUX_DATA_BASE, "sp", "The influx data base name");
- config.registerObject(BackendConfigKeys.MESSAGING_SETTINGS, MessagingSettings.fromDefault(),
+ config.registerObject(BackendConfigKeys.MESSAGING_SETTINGS, DefaultMessagingSettings.make(),
"Default Messaging Settings");
config.registerObject(BackendConfigKeys.LOCAL_AUTH_CONFIG, LocalAuthConfig.fromDefaults(getJwtSecret()),
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/DefaultMessagingSettings.java
similarity index 52%
copy from streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java
copy to streampipes-config/src/main/java/org/apache/streampipes/config/backend/DefaultMessagingSettings.java
index fe7710e88..4f65a7289 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java
+++ b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/DefaultMessagingSettings.java
@@ -15,22 +15,19 @@
* limitations under the License.
*
*/
+
package org.apache.streampipes.config.backend;
+import org.apache.streampipes.model.config.MessagingSettings;
+import org.apache.streampipes.model.config.SpDataFormat;
+import org.apache.streampipes.model.config.SpProtocol;
+
import java.util.Arrays;
import java.util.List;
-public class MessagingSettings {
-
- private Integer batchSize;
- private Integer messageMaxBytes;
- private Integer lingerMs;
- private Integer acks;
+public class DefaultMessagingSettings {
- private List<SpDataFormat> prioritizedFormats;
- private List<SpProtocol> prioritizedProtocols;
-
- public static MessagingSettings fromDefault() {
+ public static MessagingSettings make() {
List<SpProtocol> protocolList;
if (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL) != null) {
switch (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL).toLowerCase()) {
@@ -58,70 +55,4 @@ public class MessagingSettings {
Arrays.asList(SpDataFormat.JSON, SpDataFormat.CBOR, SpDataFormat.FST, SpDataFormat.SMILE),
protocolList);
}
-
- public MessagingSettings(Integer batchSize,
- Integer messageMaxBytes,
- Integer lingerMs,
- Integer acks,
- List<SpDataFormat> prioritizedFormats,
- List<SpProtocol> prioritizedProtocols) {
- this.batchSize = batchSize;
- this.messageMaxBytes = messageMaxBytes;
- this.lingerMs = lingerMs;
- this.acks = acks;
- this.prioritizedFormats = prioritizedFormats;
- this.prioritizedProtocols = prioritizedProtocols;
- }
-
- public MessagingSettings() {
-
- }
-
- public Integer getBatchSize() {
- return batchSize;
- }
-
- public void setBatchSize(Integer batchSize) {
- this.batchSize = batchSize;
- }
-
- public Integer getMessageMaxBytes() {
- return messageMaxBytes;
- }
-
- public void setMessageMaxBytes(Integer messageMaxBytes) {
- this.messageMaxBytes = messageMaxBytes;
- }
-
- public Integer getLingerMs() {
- return lingerMs;
- }
-
- public void setLingerMs(Integer lingerMs) {
- this.lingerMs = lingerMs;
- }
-
- public Integer getAcks() {
- return acks;
- }
-
- public void setAcks(Integer acks) {
- this.acks = acks;
- }
-
- public List<SpDataFormat> getPrioritizedFormats() {
- return prioritizedFormats;
- }
-
- public void setPrioritizedFormats(List<SpDataFormat> prioritizedFormats) {
- this.prioritizedFormats = prioritizedFormats;
- }
-
- public List<SpProtocol> getPrioritizedProtocols() {
- return prioritizedProtocols;
- }
-
- public void setPrioritizedProtocols(List<SpProtocol> prioritizedProtocols) {
- this.prioritizedProtocols = prioritizedProtocols;
- }
}
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java
index b62cb9351..b820956cd 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java
@@ -19,8 +19,8 @@
package org.apache.streampipes.connect.management.util;
import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpProtocol;
import org.apache.streampipes.extensions.management.connect.adapter.util.TransportFormatGenerator;
+import org.apache.streampipes.model.config.SpProtocol;
import org.apache.streampipes.model.grounding.EventGrounding;
import org.apache.streampipes.model.grounding.JmsTransportProtocol;
import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IFunctionConfig.java
similarity index 80%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
copy to streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IFunctionConfig.java
index 9f35a7bfa..16baf52ca 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
+++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IFunctionConfig.java
@@ -18,18 +18,17 @@
package org.apache.streampipes.extensions.api.declarer;
+import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.function.FunctionId;
-import java.util.List;
+import java.util.Map;
-public interface IStreamPipesFunctionDeclarer {
+public interface IFunctionConfig {
FunctionId getFunctionId();
- List<String> requiredStreamIds();
+ Map<String, SpDataStream> getOutputDataStreams();
- void invokeRuntime(String serviceGroup);
-
- void discardRuntime();
+ void addOutputDataStream(String streamId, SpDataStream stream);
}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
index 9f35a7bfa..cdf003637 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
+++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
@@ -18,13 +18,11 @@
package org.apache.streampipes.extensions.api.declarer;
-import org.apache.streampipes.model.function.FunctionId;
-
import java.util.List;
public interface IStreamPipesFunctionDeclarer {
- FunctionId getFunctionId();
+ IFunctionConfig getFunctionConfig();
List<String> requiredStreamIds();
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java
index 60d409a5e..c186688a4 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java
@@ -19,7 +19,6 @@
package org.apache.streampipes.extensions.management.connect.adapter;
import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpProtocol;
import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement;
import org.apache.streampipes.extensions.management.connect.adapter.model.pipeline.AdapterPipeline;
import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.AddTimestampPipelineElement;
@@ -34,6 +33,7 @@ import org.apache.streampipes.extensions.management.connect.adapter.preprocessin
import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.TransformStreamAdapterElement;
import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.TransformValueAdapterPipelineElement;
import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.transform.stream.DuplicateFilterPipelineElement;
+import org.apache.streampipes.model.config.SpProtocol;
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
import org.apache.streampipes.model.connect.rules.DebugSinkRuleDescription;
import org.apache.streampipes.model.connect.rules.TransformationRuleDescription;
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatGenerator.java
index 84e57d1b8..d7e0c1390 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatGenerator.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatGenerator.java
@@ -18,7 +18,7 @@
package org.apache.streampipes.extensions.management.connect.adapter.util;
import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpDataFormat;
+import org.apache.streampipes.model.config.SpDataFormat;
import org.apache.streampipes.model.grounding.TransportFormat;
import org.apache.streampipes.sdk.helpers.SupportedFormats;
import org.apache.streampipes.vocabulary.MessageFormat;
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java
index a1e04a735..32ad490dc 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java
@@ -111,7 +111,7 @@ public class DeclarersSingleton {
this.registerDataFormats(serviceDef.getDataFormatFactories());
this.allAdapters = serviceDef.getSpecificAdapters();
this.allProtocols = serviceDef.getAdapterProtocols();
- serviceDef.getFunctions().forEach(f -> this.functions.put(f.getFunctionId().getId(), f));
+ serviceDef.getFunctions().forEach(f -> this.functions.put(f.getFunctionConfig().getFunctionId().getId(), f));
}
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java b/streampipes-model/src/main/java/org/apache/streampipes/model/config/MessagingSettings.java
similarity index 66%
rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java
rename to streampipes-model/src/main/java/org/apache/streampipes/model/config/MessagingSettings.java
index fe7710e88..277aa3bf3 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/MessagingSettings.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/config/MessagingSettings.java
@@ -15,9 +15,8 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.config.backend;
+package org.apache.streampipes.model.config;
-import java.util.Arrays;
import java.util.List;
public class MessagingSettings {
@@ -30,34 +29,7 @@ public class MessagingSettings {
private List<SpDataFormat> prioritizedFormats;
private List<SpProtocol> prioritizedProtocols;
- public static MessagingSettings fromDefault() {
- List<SpProtocol> protocolList;
- if (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL) != null) {
- switch (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL).toLowerCase()) {
- case "mqtt":
- protocolList = Arrays.asList(SpProtocol.MQTT, SpProtocol.KAFKA, SpProtocol.JMS, SpProtocol.NATS);
- break;
- case "kafka":
- protocolList = Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS);
- break;
- case "jms":
- protocolList = Arrays.asList(SpProtocol.JMS, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.NATS);
- break;
- case "nats":
- protocolList = Arrays.asList(SpProtocol.NATS, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS);
- break;
- default:
- protocolList = Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS);
- }
- } else {
- protocolList = Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS);
- }
-
- return new MessagingSettings(
- 1638400, 5000012, 20, 2,
- Arrays.asList(SpDataFormat.JSON, SpDataFormat.CBOR, SpDataFormat.FST, SpDataFormat.SMILE),
- protocolList);
- }
+
public MessagingSettings(Integer batchSize,
Integer messageMaxBytes,
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpDataFormat.java b/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpDataFormat.java
similarity index 96%
rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpDataFormat.java
rename to streampipes-model/src/main/java/org/apache/streampipes/model/config/SpDataFormat.java
index 411f457cc..c409eeaab 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpDataFormat.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpDataFormat.java
@@ -15,7 +15,7 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.config.backend;
+package org.apache.streampipes.model.config;
import org.apache.streampipes.vocabulary.MessageFormat;
diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java b/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpProtocol.java
similarity index 96%
rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java
rename to streampipes-model/src/main/java/org/apache/streampipes/model/config/SpProtocol.java
index b69ce31fb..bd6f2a4e9 100644
--- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpProtocol.java
@@ -15,7 +15,7 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.config.backend;
+package org.apache.streampipes.model.config;
public enum SpProtocol {
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/DataSetGroundingSelector.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/DataSetGroundingSelector.java
index 75e6f5383..1a419b747 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/DataSetGroundingSelector.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/DataSetGroundingSelector.java
@@ -18,9 +18,9 @@
package org.apache.streampipes.manager.matching;
import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpProtocol;
import org.apache.streampipes.manager.util.TopicGenerator;
import org.apache.streampipes.model.SpDataSet;
+import org.apache.streampipes.model.config.SpProtocol;
import org.apache.streampipes.model.grounding.EventGrounding;
import org.apache.streampipes.model.grounding.JmsTransportProtocol;
import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java
index ba9b3d345..128098f7c 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java
@@ -19,10 +19,10 @@
package org.apache.streampipes.manager.matching;
import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpDataFormat;
import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.model.config.SpDataFormat;
import org.apache.streampipes.model.grounding.TransportFormat;
import org.apache.streampipes.vocabulary.MessageFormat;
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java
index 48e211cb8..9631a4cad 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java
@@ -19,11 +19,11 @@
package org.apache.streampipes.manager.matching;
import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.SpProtocol;
import org.apache.streampipes.manager.util.TopicGenerator;
import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.model.config.SpProtocol;
import org.apache.streampipes.model.grounding.JmsTransportProtocol;
import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
import org.apache.streampipes.model.grounding.MqttTransportProtocol;
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java
index 87b3d68f4..17bdb22a8 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java
@@ -20,7 +20,7 @@ package org.apache.streampipes.rest.impl.admin;
import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.config.backend.MessagingSettings;
+import org.apache.streampipes.model.config.MessagingSettings;
import org.apache.streampipes.rest.core.base.impl.AbstractRestResource;
import org.apache.streampipes.rest.security.AuthConstants;
import org.apache.streampipes.rest.shared.annotation.JacksonSerialized;
diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/StreamPipesFunctionHandler.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/StreamPipesFunctionHandler.java
index 46d8f135f..a9ec6cc25 100644
--- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/StreamPipesFunctionHandler.java
+++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/StreamPipesFunctionHandler.java
@@ -18,8 +18,11 @@
package org.apache.streampipes.service.extensions.function;
+import org.apache.streampipes.extensions.api.declarer.DataStreamDeclarer;
import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer;
+import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver;
import org.apache.streampipes.extensions.management.init.DeclarersSingleton;
+import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.function.FunctionDefinition;
import java.util.Collection;
@@ -44,15 +47,40 @@ public enum StreamPipesFunctionHandler {
public void initializeFunctions(Collection<IStreamPipesFunctionDeclarer> functions,
String serviceGroup) {
+ registerDataStreams(functions);
functions.forEach(function -> {
function.invokeRuntime(serviceGroup);
- runningInstances.put(function.getFunctionId().getId(), function);
+ runningInstances.put(function.getFunctionConfig().getFunctionId().getId(), function);
});
if (!(runningInstances.isEmpty())) {
new Thread(new FunctionRegistrationHandler(getFunctionDefinitions())).start();
}
}
+ public void registerDataStreams(Collection<IStreamPipesFunctionDeclarer> functions) {
+ var client = new StreamPipesClientResolver().makeStreamPipesClientInstance();
+ functions.forEach(function -> {
+ function.getFunctionConfig().getOutputDataStreams().values().forEach(ds -> {
+ DeclarersSingleton.getInstance().add(new DataStreamDeclarer() {
+ @Override
+ public SpDataStream declareModel() {
+ return ds;
+ }
+
+ @Override
+ public void executeStream() {
+
+ }
+
+ @Override
+ public boolean isExecutable() {
+ return false;
+ }
+ });
+ });
+ });
+ }
+
public void cleanupFunctions() {
this.runningInstances.forEach((key, value) -> {
value.discardRuntime();
@@ -64,7 +92,9 @@ public enum StreamPipesFunctionHandler {
return this.runningInstances
.values()
.stream()
- .map(function -> new FunctionDefinition(function.getFunctionId(), function.requiredStreamIds()))
+ .map(function -> new FunctionDefinition(
+ function.getFunctionConfig().getFunctionId(),
+ function.requiredStreamIds()))
.collect(Collectors.toList());
}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfig.java
similarity index 50%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
copy to streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfig.java
index 9f35a7bfa..3e7448c07 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfig.java
@@ -16,20 +16,36 @@
*
*/
-package org.apache.streampipes.extensions.api.declarer;
+package org.apache.streampipes.wrapper.standalone.function;
+import org.apache.streampipes.extensions.api.declarer.IFunctionConfig;
+import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.function.FunctionId;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
-public interface IStreamPipesFunctionDeclarer {
+public class FunctionConfig implements IFunctionConfig {
- FunctionId getFunctionId();
+ private final FunctionId functionId;
+ private final Map<String, SpDataStream> outputDataStreams;
- List<String> requiredStreamIds();
+ public FunctionConfig(FunctionId functionId) {
+ this.functionId = functionId;
+ this.outputDataStreams = new HashMap<>();
+ }
- void invokeRuntime(String serviceGroup);
+ public FunctionId getFunctionId() {
+ return functionId;
+ }
- void discardRuntime();
+ public Map<String, SpDataStream> getOutputDataStreams() {
+ return outputDataStreams;
+ }
+
+ public void addOutputDataStream(String streamId,
+ SpDataStream stream) {
+ this.outputDataStreams.put(streamId, stream);
+ }
}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfigBuilder.java
similarity index 56%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
copy to streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfigBuilder.java
index 9f35a7bfa..0d2c8e246 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/IStreamPipesFunctionDeclarer.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionConfigBuilder.java
@@ -16,20 +16,31 @@
*
*/
-package org.apache.streampipes.extensions.api.declarer;
+package org.apache.streampipes.wrapper.standalone.function;
+import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.function.FunctionId;
-import java.util.List;
+public class FunctionConfigBuilder {
-public interface IStreamPipesFunctionDeclarer {
+ private final FunctionConfig functionConfig;
- FunctionId getFunctionId();
+ public static FunctionConfigBuilder create(FunctionId functionId) {
+ return new FunctionConfigBuilder(functionId);
+ }
- List<String> requiredStreamIds();
+ private FunctionConfigBuilder(FunctionId functionId) {
+ this.functionConfig = new FunctionConfig(functionId);
+ }
- void invokeRuntime(String serviceGroup);
+ public FunctionConfigBuilder withOutputStream(SpDataStream stream) {
+ this.functionConfig.addOutputDataStream(stream.getAppId(), stream);
+ return this;
+ }
+
+ public FunctionConfig build() {
+ return functionConfig;
+ }
- void discardRuntime();
}
diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java
index dc8370605..5d015c64b 100644
--- a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.extensions.management.monitoring.SpMonitoringManag
import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.monitoring.SpLogEntry;
import org.apache.streampipes.model.schema.EventSchema;
+import org.apache.streampipes.wrapper.routing.SpOutputCollector;
import java.util.Collection;
import java.util.HashMap;
@@ -33,10 +34,13 @@ import java.util.Map;
public class FunctionContext {
private final Map<String, SpDataStream> streams;
+
private StreamPipesClient client;
private String functionId;
private ConfigExtractor config;
+ private Map<String, SpOutputCollector> outputCollectors;
+
public FunctionContext() {
this.streams = new HashMap<>();
}
@@ -44,11 +48,13 @@ public class FunctionContext {
public FunctionContext(String functionId,
String serviceGroup,
List<SpDataStream> streams,
+ Map<String, SpOutputCollector> outputCollectors,
StreamPipesClient client) {
this();
streams.forEach(stream -> this.streams.put(stream.getElementId(), stream));
this.config = ConfigExtractor.from(serviceGroup);
this.functionId = functionId;
+ this.outputCollectors = outputCollectors;
this.client = client;
}
@@ -76,4 +82,7 @@ public class FunctionContext {
SpMonitoringManager.INSTANCE.addErrorMessage(functionId, logEntry);
}
+ public Map<String, SpOutputCollector> getOutputCollectors() {
+ return outputCollectors;
+ }
}
diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java
index 483f712c1..c3465b413 100644
--- a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java
@@ -21,8 +21,10 @@ package org.apache.streampipes.wrapper.standalone.function;
import org.apache.streampipes.client.StreamPipesClient;
import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver;
import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.wrapper.routing.SpOutputCollector;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
public class FunctionContextGenerator {
@@ -31,19 +33,23 @@ public class FunctionContextGenerator {
private final String functionId;
private final String serviceGroup;
+ private final Map<String, SpOutputCollector> outputCollectors;
+
public FunctionContextGenerator(String functionId,
String serviceGroup,
- List<String> streamIds) {
+ List<String> streamIds,
+ Map<String, SpOutputCollector> outputCollectors) {
this.streamIds = streamIds;
this.serviceGroup = serviceGroup;
this.functionId = functionId;
+ this.outputCollectors = outputCollectors;
}
public FunctionContext generate() {
var client = makeClient();
var streams = receiveStreams(client);
- return new FunctionContext(functionId, serviceGroup, streams, client);
+ return new FunctionContext(functionId, serviceGroup, streams, outputCollectors, client);
}
private StreamPipesClient makeClient() {
diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java
index 547dc2dfe..8f79526e2 100644
--- a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.wrapper.standalone.function;
import org.apache.streampipes.commons.constants.Envs;
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.extensions.api.declarer.IFunctionConfig;
import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer;
import org.apache.streampipes.extensions.management.monitoring.SpMonitoringManager;
import org.apache.streampipes.extensions.management.util.GroundingDebugUtils;
@@ -34,6 +35,7 @@ import org.apache.streampipes.model.runtime.SourceInfo;
import org.apache.streampipes.model.schema.EventSchema;
import org.apache.streampipes.wrapper.routing.RawDataProcessor;
import org.apache.streampipes.wrapper.routing.SpInputCollector;
+import org.apache.streampipes.wrapper.routing.SpOutputCollector;
import org.apache.streampipes.wrapper.standalone.manager.ProtocolManager;
import org.slf4j.Logger;
@@ -52,16 +54,23 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
private final Map<String, SchemaInfo> schemaInfoMapper;
private Map<String, SpInputCollector> inputCollectors;
+ private Map<String, SpOutputCollector> outputCollectors;
+
public StreamPipesFunction() {
this.sourceInfoMapper = new HashMap<>();
this.schemaInfoMapper = new HashMap<>();
+ this.outputCollectors = new HashMap<>();
}
@Override
public void invokeRuntime(String serviceGroup) {
+ var functionId = this.getFunctionConfig().getFunctionId();
+ this.outputCollectors = this.getOutputCollectors();
+ this.outputCollectors.forEach((key, value) -> value.connect());
+
FunctionContext context = new FunctionContextGenerator(
- this.getFunctionId().getId(), serviceGroup, this.requiredStreamIds()
- ).generate();
+ functionId.getId(), serviceGroup, this.requiredStreamIds(), this.outputCollectors)
+ .generate();
// Creates a source info for each incoming SpDataStream
// The index is used to create the selector prefix for the SourceInfo
@@ -76,15 +85,18 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
});
this.inputCollectors = getInputCollectors(context.getStreams());
- LOG.info("Invoking function {}:{}", this.getFunctionId().getId(), this.getFunctionId().getVersion());
+
+ LOG.info("Invoking function {}:{}", functionId.getId(), functionId.getVersion());
onServiceStarted(context);
registerConsumers();
}
public void discardRuntime() {
- LOG.info("Discarding function {}:{}", this.getFunctionId().getId(), this.getFunctionId().getVersion());
+ var functionId = this.getFunctionConfig().getFunctionId();
+ LOG.info("Discarding function {}:{}", functionId.getId(), functionId.getVersion());
onServiceStopped();
unregisterConsumers();
+ this.outputCollectors.forEach((key, value) -> value.disconnect());
}
@Override
@@ -107,19 +119,34 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
}
private void increaseCounter(String sourceInfo) {
+ var functionId = this.getFunctionConfig().getFunctionId();
SpMonitoringManager.INSTANCE.increaseInCounter(
- this.getFunctionId().getId(),
+ functionId.getId(),
sourceInfo,
System.currentTimeMillis()
);
}
private void addError(RuntimeException e) {
+ var functionId = this.getFunctionConfig().getFunctionId();
SpMonitoringManager.INSTANCE.addErrorMessage(
- this.getFunctionId().getId(),
+ functionId.getId(),
SpLogEntry.from(System.currentTimeMillis(), StreamPipesErrorMessage.from(e)));
}
+ private Map<String, SpOutputCollector> getOutputCollectors() {
+ this.getFunctionConfig().getOutputDataStreams().forEach((key, value) -> {
+ this.outputCollectors.put(
+ key,
+ ProtocolManager.makeOutputCollector(
+ value.getEventGrounding().getTransportProtocol(),
+ value.getEventGrounding().getTransportFormats().get(0),
+ key));
+ });
+
+ return this.outputCollectors;
+ }
+
private Map<String, SpInputCollector> getInputCollectors(Collection<SpDataStream> streams) throws SpRuntimeException {
Map<String, SpInputCollector> inputCollectors = new HashMap<>();
for (SpDataStream is : streams) {
@@ -146,13 +173,6 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
});
}
- public abstract void onServiceStarted(FunctionContext context);
-
- public abstract void onEvent(Event event,
- String streamId);
-
- public abstract void onServiceStopped();
-
private SourceInfo createSourceInfo(SpDataStream stream, int streamIndex) {
return new SourceInfo(
stream.getElementId(),
@@ -164,4 +184,13 @@ public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclare
return new SchemaInfo(eventSchema, new ArrayList<>());
}
+ public abstract IFunctionConfig getFunctionConfig();
+
+ public abstract void onServiceStarted(FunctionContext context);
+
+ public abstract void onEvent(Event event,
+ String streamId);
+
+ public abstract void onServiceStopped();
+
}