You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2022/10/23 21:43:15 UTC
[incubator-streampipes] 01/03: [STREAMPIPES-603] Add standalone functions
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit c06c51ed29e047a04a917bda9296cc0ae5b7a1ae
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Sun Oct 23 16:35:33 2022 +0200
[STREAMPIPES-603] Add standalone functions
---
.../streampipes/client/api/DataStreamApi.java | 3 +-
.../commons/constants/DefaultEnvValues.java | 1 +
.../apache/streampipes/commons/constants/Envs.java | 7 +-
.../extensions/ExtensionsModelSubmitter.java | 3 +
.../function/StreamPipesFunctionHandler.java | 50 ++++++++
.../PipelineElementServiceShutdownHandler.java | 1 +
.../api/DataProcessorPipelineElementResource.java | 18 +--
.../api/DataSinkPipelineElementResource.java | 12 +-
.../declarer/IStreamPipesFunctionDeclarer.java | 22 ++--
.../container/init/DeclarersSingleton.java | 7 ++
.../container/model/SpServiceDefinition.java | 11 ++
.../model/SpServiceDefinitionBuilder.java | 6 +
.../container/util/GroundingDebugUtils.java | 21 ++--
.../extensions/all/iiot/AllExtensionsIIoTInit.java | 48 ++++----
.../streampipes/model/function/FunctionId.java | 31 +++--
.../standalone/function/FunctionContext.java | 67 +++++++++++
.../function/FunctionContextGenerator.java | 56 +++++++++
.../standalone/function/StreamPipesFunction.java | 129 +++++++++++++++++++++
18 files changed, 421 insertions(+), 72 deletions(-)
diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/DataStreamApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/DataStreamApi.java
index 3ed9d2c22..8bf016bd4 100644
--- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/DataStreamApi.java
+++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/DataStreamApi.java
@@ -37,7 +37,8 @@ public class DataStreamApi extends AbstractTypedClientApi<SpDataStream> implemen
@Override
public SpDataStream get(String streamId) {
- return getSingle(getBaseResourcePath().addToPath(streamId));
+ return getSingle(StreamPipesApiPath.fromBaseApiPath()
+ .addToPath("streams").addToPath(streamId));
}
/**
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
index 5a7b2f54b..ec49279fa 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
@@ -23,6 +23,7 @@ public class DefaultEnvValues {
public static final String INITIAL_ADMIN_PW_DEFAULT = "admin";
public static final String INITIAL_CLIENT_USER_DEFAULT = "sp-service-client";
public static final String INITIAL_CLIENT_SECRET_DEFAULT = "my-apache-streampipes-secret-key-change-me";
+ public static final int MAX_WAIT_TIME_AT_SHUTDOWN_DEFAULT = 10000;
public static final boolean INSTALL_PIPELINE_ELEMENTS = true;
public static final String DEFAULT_ENCRYPTION_PASSCODE = "eGgemyGBoILAu3xckoIp";
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
index 217f649fd..9543bc62f 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
@@ -36,7 +36,8 @@ public enum Envs {
SP_CLIENT_USER("SP_CLIENT_USER"),
SP_CLIENT_SECRET("SP_CLIENT_SECRET"),
SP_ENCRYPTION_PASSCODE("SP_ENCRYPTION_PASSCODE"),
- SP_DEBUG("SP_DEBUG");
+ SP_DEBUG("SP_DEBUG"),
+ SP_MAX_WAIT_TIME_AT_SHUTDOWN("SP_MAX_WAIT_TIME_AT_SHUTDOWN");
private final String envVariableName;
@@ -56,6 +57,10 @@ public enum Envs {
return CustomEnvs.getEnvAsInt(this.envVariableName);
}
+ public Integer getValueAsIntOrDefault(int defaultValue) {
+ return exists() ? getValueAsInt() : defaultValue;
+ }
+
public Boolean getValueAsBoolean() {
return CustomEnvs.getEnvAsBoolean(this.envVariableName);
}
diff --git a/streampipes-container-extensions/src/main/java/org/apache/streampipes/container/extensions/ExtensionsModelSubmitter.java b/streampipes-container-extensions/src/main/java/org/apache/streampipes/container/extensions/ExtensionsModelSubmitter.java
index d0e97de0a..7ea994097 100644
--- a/streampipes-container-extensions/src/main/java/org/apache/streampipes/container/extensions/ExtensionsModelSubmitter.java
+++ b/streampipes-container-extensions/src/main/java/org/apache/streampipes/container/extensions/ExtensionsModelSubmitter.java
@@ -19,6 +19,7 @@ package org.apache.streampipes.container.extensions;
import org.apache.streampipes.connect.container.worker.init.ConnectWorkerRegistrationService;
import org.apache.streampipes.connect.container.worker.init.ConnectWorkerTagProvider;
+import org.apache.streampipes.container.extensions.function.StreamPipesFunctionHandler;
import org.apache.streampipes.container.init.DeclarersSingleton;
import org.apache.streampipes.container.model.SpServiceDefinition;
import org.apache.streampipes.container.standalone.init.PipelineElementServiceShutdownHandler;
@@ -46,12 +47,14 @@ public abstract class ExtensionsModelSubmitter extends StreamPipesExtensionsServ
@PreDestroy
public void onExit() {
new PipelineElementServiceShutdownHandler().onShutdown();
+ StreamPipesFunctionHandler.INSTANCE.cleanupFunctions();
deregisterService(DeclarersSingleton.getInstance().getServiceId());
}
@Override
public void afterServiceRegistered(SpServiceDefinition serviceDef) {
new ConnectWorkerRegistrationService().registerWorker(serviceDef);
+ StreamPipesFunctionHandler.INSTANCE.initializeFunctions();
}
@Override
diff --git a/streampipes-container-extensions/src/main/java/org/apache/streampipes/container/extensions/function/StreamPipesFunctionHandler.java b/streampipes-container-extensions/src/main/java/org/apache/streampipes/container/extensions/function/StreamPipesFunctionHandler.java
new file mode 100644
index 000000000..d4985decf
--- /dev/null
+++ b/streampipes-container-extensions/src/main/java/org/apache/streampipes/container/extensions/function/StreamPipesFunctionHandler.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.container.extensions.function;
+
+import org.apache.streampipes.container.declarer.IStreamPipesFunctionDeclarer;
+import org.apache.streampipes.container.init.DeclarersSingleton;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public enum StreamPipesFunctionHandler {
+
+ INSTANCE;
+
+ private final Map<String, IStreamPipesFunctionDeclarer> runningInstances;
+
+ StreamPipesFunctionHandler() {
+ this.runningInstances = new HashMap<>();
+ }
+
+ public void initializeFunctions() {
+ DeclarersSingleton.getInstance().getFunctions().forEach((key, value) -> {
+ Runnable r = value::invokeRuntime;
+ new Thread(r).start();
+ runningInstances.put(key, value);
+ });
+ }
+
+ public void cleanupFunctions() {
+ this.runningInstances.forEach((key, value) -> {
+ value.discardRuntime();
+ });
+ }
+}
diff --git a/streampipes-container-standalone/src/main/java/org/apache/streampipes/container/standalone/init/PipelineElementServiceShutdownHandler.java b/streampipes-container-standalone/src/main/java/org/apache/streampipes/container/standalone/init/PipelineElementServiceShutdownHandler.java
index 2faa9978e..6b3522d5e 100644
--- a/streampipes-container-standalone/src/main/java/org/apache/streampipes/container/standalone/init/PipelineElementServiceShutdownHandler.java
+++ b/streampipes-container-standalone/src/main/java/org/apache/streampipes/container/standalone/init/PipelineElementServiceShutdownHandler.java
@@ -25,6 +25,7 @@ public class PipelineElementServiceShutdownHandler {
private static final Logger LOG = LoggerFactory.getLogger(PipelineElementServiceShutdownHandler.class);
+
public void onShutdown() {
LOG.info("Shutting down StreamPipes pipeline element container...");
int runningInstancesCount = RunningInstances.INSTANCE.getRunningInstancesCount();
diff --git a/streampipes-container/src/main/java/org/apache/streampipes/container/api/DataProcessorPipelineElementResource.java b/streampipes-container/src/main/java/org/apache/streampipes/container/api/DataProcessorPipelineElementResource.java
index 3d333487d..cf4916205 100644
--- a/streampipes-container/src/main/java/org/apache/streampipes/container/api/DataProcessorPipelineElementResource.java
+++ b/streampipes-container/src/main/java/org/apache/streampipes/container/api/DataProcessorPipelineElementResource.java
@@ -19,14 +19,12 @@
package org.apache.streampipes.container.api;
import org.apache.streampipes.commons.constants.InstanceIdExtractor;
-import org.apache.streampipes.svcdiscovery.api.model.SpServicePathPrefix;
import org.apache.streampipes.container.declarer.SemanticEventProcessingAgentDeclarer;
import org.apache.streampipes.container.init.DeclarersSingleton;
+import org.apache.streampipes.container.util.GroundingDebugUtils;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
-import org.apache.streampipes.model.grounding.EventGrounding;
-import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
-import org.apache.streampipes.model.grounding.TransportProtocol;
import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.apache.streampipes.svcdiscovery.api.model.SpServicePathPrefix;
import javax.ws.rs.Path;
import java.util.Map;
@@ -58,18 +56,10 @@ public class DataProcessorPipelineElementResource extends InvocablePipelineEleme
@Override
protected DataProcessorInvocation createGroundingDebugInformation(DataProcessorInvocation graph) {
graph.getInputStreams().forEach(is -> {
- modifyGrounding(is.getEventGrounding());
+ GroundingDebugUtils.modifyGrounding(is.getEventGrounding());
});
- modifyGrounding(graph.getOutputStream().getEventGrounding());
+ GroundingDebugUtils.modifyGrounding(graph.getOutputStream().getEventGrounding());
return graph;
}
-
- private void modifyGrounding(EventGrounding grounding) {
- TransportProtocol protocol = grounding.getTransportProtocol();
- protocol.setBrokerHostname("localhost");
- if (protocol instanceof KafkaTransportProtocol) {
- ((KafkaTransportProtocol) protocol).setKafkaPort(9094);
- }
- }
}
diff --git a/streampipes-container/src/main/java/org/apache/streampipes/container/api/DataSinkPipelineElementResource.java b/streampipes-container/src/main/java/org/apache/streampipes/container/api/DataSinkPipelineElementResource.java
index 130321687..67483985b 100644
--- a/streampipes-container/src/main/java/org/apache/streampipes/container/api/DataSinkPipelineElementResource.java
+++ b/streampipes-container/src/main/java/org/apache/streampipes/container/api/DataSinkPipelineElementResource.java
@@ -19,13 +19,12 @@
package org.apache.streampipes.container.api;
import org.apache.streampipes.commons.constants.InstanceIdExtractor;
-import org.apache.streampipes.svcdiscovery.api.model.SpServicePathPrefix;
import org.apache.streampipes.container.declarer.SemanticEventConsumerDeclarer;
import org.apache.streampipes.container.init.DeclarersSingleton;
+import org.apache.streampipes.container.util.GroundingDebugUtils;
import org.apache.streampipes.model.graph.DataSinkInvocation;
-import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
-import org.apache.streampipes.model.grounding.TransportProtocol;
import org.apache.streampipes.sdk.extractor.DataSinkParameterExtractor;
+import org.apache.streampipes.svcdiscovery.api.model.SpServicePathPrefix;
import javax.ws.rs.Path;
import java.util.Map;
@@ -57,12 +56,7 @@ public class DataSinkPipelineElementResource extends InvocablePipelineElementRes
@Override
protected DataSinkInvocation createGroundingDebugInformation(DataSinkInvocation graph) {
graph.getInputStreams().forEach(is -> {
- TransportProtocol protocol = is.getEventGrounding().getTransportProtocol();
- protocol.setBrokerHostname("localhost");
- if (protocol instanceof KafkaTransportProtocol) {
- ((KafkaTransportProtocol) protocol).setKafkaPort(9094);
- }
-
+ GroundingDebugUtils.modifyGrounding(is.getEventGrounding());
});
return graph;
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java b/streampipes-container/src/main/java/org/apache/streampipes/container/declarer/IStreamPipesFunctionDeclarer.java
similarity index 58%
copy from streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
copy to streampipes-container/src/main/java/org/apache/streampipes/container/declarer/IStreamPipesFunctionDeclarer.java
index 5a7b2f54b..30f500e38 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
+++ b/streampipes-container/src/main/java/org/apache/streampipes/container/declarer/IStreamPipesFunctionDeclarer.java
@@ -15,15 +15,21 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.commons.constants;
-public class DefaultEnvValues {
+package org.apache.streampipes.container.declarer;
- public static final String INITIAL_ADMIN_EMAIL_DEFAULT = "admin@streampipes.apache.org";
- public static final String INITIAL_ADMIN_PW_DEFAULT = "admin";
- public static final String INITIAL_CLIENT_USER_DEFAULT = "sp-service-client";
- public static final String INITIAL_CLIENT_SECRET_DEFAULT = "my-apache-streampipes-secret-key-change-me";
- public static final boolean INSTALL_PIPELINE_ELEMENTS = true;
+import org.apache.streampipes.model.function.FunctionId;
+
+import java.util.List;
+
+public interface IStreamPipesFunctionDeclarer {
+
+ FunctionId getFunctionId();
+
+ List<String> requiredStreamIds();
+
+ void invokeRuntime();
+
+ void discardRuntime();
- public static final String DEFAULT_ENCRYPTION_PASSCODE = "eGgemyGBoILAu3xckoIp";
}
diff --git a/streampipes-container/src/main/java/org/apache/streampipes/container/init/DeclarersSingleton.java b/streampipes-container/src/main/java/org/apache/streampipes/container/init/DeclarersSingleton.java
index 936bb223e..967ae027e 100644
--- a/streampipes-container/src/main/java/org/apache/streampipes/container/init/DeclarersSingleton.java
+++ b/streampipes-container/src/main/java/org/apache/streampipes/container/init/DeclarersSingleton.java
@@ -57,6 +57,7 @@ public class DeclarersSingleton {
private Map<String, SemanticEventConsumerDeclarer> consumerDeclarers;
private Map<String, PipelineTemplateDeclarer> pipelineTemplateDeclarers;
private Map<String, DataStreamDeclarer> streamDeclarers;
+ private Map<String, IStreamPipesFunctionDeclarer> functions;
private Map<String, TransportProtocol> supportedProtocols;
private Map<String, TransportFormat> supportedFormats;
@@ -80,6 +81,7 @@ public class DeclarersSingleton {
this.supportedFormats = new HashMap<>();
this.allProtocols = new HashMap<>();
this.allAdapters = new HashMap<>();
+ this.functions = new HashMap<>();
this.route = "/";
}
@@ -102,6 +104,7 @@ public class DeclarersSingleton {
this.registerDataFormats(serviceDef.getDataFormatFactories());
this.allAdapters = serviceDef.getSpecificAdapters();
this.allProtocols = serviceDef.getAdapterProtocols();
+ serviceDef.getFunctions().forEach(f -> this.functions.put(f.getFunctionId().getId(), f));
}
@@ -312,6 +315,10 @@ public class DeclarersSingleton {
return serviceId;
}
+ public Map<String, IStreamPipesFunctionDeclarer> getFunctions() {
+ return functions;
+ }
+
private void checkAndStartExecutableStreams(DataStreamDeclarer declarer) {
if (declarer.isExecutable()) {
declarer.executeStream();
diff --git a/streampipes-container/src/main/java/org/apache/streampipes/container/model/SpServiceDefinition.java b/streampipes-container/src/main/java/org/apache/streampipes/container/model/SpServiceDefinition.java
index dfe8e3f5f..14e8e8394 100644
--- a/streampipes-container/src/main/java/org/apache/streampipes/container/model/SpServiceDefinition.java
+++ b/streampipes-container/src/main/java/org/apache/streampipes/container/model/SpServiceDefinition.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.container.model;
import org.apache.streampipes.connect.api.IAdapter;
import org.apache.streampipes.connect.api.IProtocol;
import org.apache.streampipes.container.declarer.Declarer;
+import org.apache.streampipes.container.declarer.IStreamPipesFunctionDeclarer;
import org.apache.streampipes.dataformat.SpDataFormatFactory;
import org.apache.streampipes.messaging.SpProtocolDefinitionFactory;
import org.apache.streampipes.svcdiscovery.api.model.ConfigItem;
@@ -37,6 +38,7 @@ public class SpServiceDefinition {
private List<Declarer<?>> declarers;
private List<SpDataFormatFactory> dataFormatFactories;
private List<SpProtocolDefinitionFactory<?>> protocolDefinitionFactories;
+ private List<IStreamPipesFunctionDeclarer> functions;
private Map<String, IProtocol> adapterProtocols;
private Map<String, IAdapter> specificAdapters;
@@ -50,6 +52,7 @@ public class SpServiceDefinition {
this.adapterProtocols = new HashMap<>();
this.specificAdapters = new HashMap<>();
this.kvConfigs = new HashMap<>();
+ this.functions = new ArrayList<>();
}
public String getServiceGroup() {
@@ -171,4 +174,12 @@ public class SpServiceDefinition {
public Map<String, ConfigItem> getKvConfigs() {
return kvConfigs;
}
+
+ public void addStreamPipesFunction(IStreamPipesFunctionDeclarer function) {
+ this.functions.add(function);
+ }
+
+ public List<IStreamPipesFunctionDeclarer> getFunctions() {
+ return functions;
+ }
}
diff --git a/streampipes-container/src/main/java/org/apache/streampipes/container/model/SpServiceDefinitionBuilder.java b/streampipes-container/src/main/java/org/apache/streampipes/container/model/SpServiceDefinitionBuilder.java
index 0e0f2900e..d21f3e490 100644
--- a/streampipes-container/src/main/java/org/apache/streampipes/container/model/SpServiceDefinitionBuilder.java
+++ b/streampipes-container/src/main/java/org/apache/streampipes/container/model/SpServiceDefinitionBuilder.java
@@ -21,6 +21,7 @@ import org.apache.streampipes.connect.api.Connector;
import org.apache.streampipes.connect.api.IAdapter;
import org.apache.streampipes.connect.api.IProtocol;
import org.apache.streampipes.container.declarer.Declarer;
+import org.apache.streampipes.container.declarer.IStreamPipesFunctionDeclarer;
import org.apache.streampipes.dataformat.SpDataFormatFactory;
import org.apache.streampipes.messaging.SpProtocolDefinitionFactory;
import org.apache.streampipes.svcdiscovery.api.model.ConfigItem;
@@ -143,6 +144,11 @@ public class SpServiceDefinitionBuilder {
return this;
}
+ public SpServiceDefinitionBuilder registerFunction(IStreamPipesFunctionDeclarer function) {
+ this.serviceDefinition.addStreamPipesFunction(function);
+ return this;
+ }
+
public SpServiceDefinition build() {
return this.serviceDefinition;
}
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java b/streampipes-container/src/main/java/org/apache/streampipes/container/util/GroundingDebugUtils.java
similarity index 58%
copy from streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
copy to streampipes-container/src/main/java/org/apache/streampipes/container/util/GroundingDebugUtils.java
index 5a7b2f54b..b21b8d0a5 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
+++ b/streampipes-container/src/main/java/org/apache/streampipes/container/util/GroundingDebugUtils.java
@@ -15,15 +15,20 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.commons.constants;
-public class DefaultEnvValues {
+package org.apache.streampipes.container.util;
- public static final String INITIAL_ADMIN_EMAIL_DEFAULT = "admin@streampipes.apache.org";
- public static final String INITIAL_ADMIN_PW_DEFAULT = "admin";
- public static final String INITIAL_CLIENT_USER_DEFAULT = "sp-service-client";
- public static final String INITIAL_CLIENT_SECRET_DEFAULT = "my-apache-streampipes-secret-key-change-me";
- public static final boolean INSTALL_PIPELINE_ELEMENTS = true;
+import org.apache.streampipes.model.grounding.EventGrounding;
+import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
+import org.apache.streampipes.model.grounding.TransportProtocol;
- public static final String DEFAULT_ENCRYPTION_PASSCODE = "eGgemyGBoILAu3xckoIp";
+public class GroundingDebugUtils {
+
+ public static void modifyGrounding(EventGrounding grounding) {
+ TransportProtocol protocol = grounding.getTransportProtocol();
+ protocol.setBrokerHostname("localhost");
+ if (protocol instanceof KafkaTransportProtocol) {
+ ((KafkaTransportProtocol) protocol).setKafkaPort(9094);
+ }
+ }
}
diff --git a/streampipes-extensions/streampipes-extensions-all-iiot/src/main/java/org/apache/streampipes/extensions/all/iiot/AllExtensionsIIoTInit.java b/streampipes-extensions/streampipes-extensions-all-iiot/src/main/java/org/apache/streampipes/extensions/all/iiot/AllExtensionsIIoTInit.java
index d58da6ed7..28aea5da3 100644
--- a/streampipes-extensions/streampipes-extensions-all-iiot/src/main/java/org/apache/streampipes/extensions/all/iiot/AllExtensionsIIoTInit.java
+++ b/streampipes-extensions/streampipes-extensions-all-iiot/src/main/java/org/apache/streampipes/extensions/all/iiot/AllExtensionsIIoTInit.java
@@ -29,6 +29,7 @@ import org.apache.streampipes.dataformat.smile.SmileDataFormatFactory;
import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory;
import org.apache.streampipes.messaging.kafka.SpKafkaProtocolFactory;
import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory;
+import org.apache.streampipes.messaging.nats.SpNatsProtocolFactory;
import org.apache.streampipes.processors.changedetection.jvm.ChangeDetectionJvmInit;
import org.apache.streampipes.processors.enricher.jvm.EnricherJvmInit;
import org.apache.streampipes.processors.filters.jvm.FiltersJvmInit;
@@ -41,34 +42,35 @@ import org.apache.streampipes.sinks.notifications.jvm.SinksNotificationsJvmInit;
public class AllExtensionsIIoTInit extends ExtensionsModelSubmitter {
- public static void main (String[] args) {
+ public static void main(String[] args) {
new AllExtensionsIIoTInit().init();
}
@Override
public SpServiceDefinition provideServiceDefinition() {
return SpServiceDefinitionBuilder.create("org.apache.streampipes.extensions.all.iiot",
- "StreamPipes Extensions (IIoT only)",
- "", 8090)
- .merge(new ConnectAdapterIiotInit().provideServiceDefinition())
- .merge(new SinksInternalJvmInit().provideServiceDefinition())
- .merge(new FiltersJvmInit().provideServiceDefinition())
- .merge(new ChangeDetectionJvmInit().provideServiceDefinition())
- .merge(new EnricherJvmInit().provideServiceDefinition())
- .merge(new FiltersSiddhiInit().provideServiceDefinition())
- .merge(new TransformationJvmInit().provideServiceDefinition())
- .merge(new BrokersJvmInit().provideServiceDefinition())
- .merge(new DatabasesJvmInit().provideServiceDefinition())
- .merge(new SinksNotificationsJvmInit().provideServiceDefinition())
- .registerMessagingFormats(
- new JsonDataFormatFactory(),
- new CborDataFormatFactory(),
- new SmileDataFormatFactory(),
- new FstDataFormatFactory())
- .registerMessagingProtocols(
- new SpKafkaProtocolFactory(),
- new SpJmsProtocolFactory(),
- new SpMqttProtocolFactory())
- .build();
+ "StreamPipes Extensions (IIoT only)",
+ "", 8090)
+ .merge(new ConnectAdapterIiotInit().provideServiceDefinition())
+ .merge(new SinksInternalJvmInit().provideServiceDefinition())
+ .merge(new FiltersJvmInit().provideServiceDefinition())
+ .merge(new ChangeDetectionJvmInit().provideServiceDefinition())
+ .merge(new EnricherJvmInit().provideServiceDefinition())
+ .merge(new FiltersSiddhiInit().provideServiceDefinition())
+ .merge(new TransformationJvmInit().provideServiceDefinition())
+ .merge(new BrokersJvmInit().provideServiceDefinition())
+ .merge(new DatabasesJvmInit().provideServiceDefinition())
+ .merge(new SinksNotificationsJvmInit().provideServiceDefinition())
+ .registerMessagingFormats(
+ new JsonDataFormatFactory(),
+ new CborDataFormatFactory(),
+ new SmileDataFormatFactory(),
+ new FstDataFormatFactory())
+ .registerMessagingProtocols(
+ new SpKafkaProtocolFactory(),
+ new SpJmsProtocolFactory(),
+ new SpMqttProtocolFactory(),
+ new SpNatsProtocolFactory())
+ .build();
}
}
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java b/streampipes-model/src/main/java/org/apache/streampipes/model/function/FunctionId.java
similarity index 58%
copy from streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
copy to streampipes-model/src/main/java/org/apache/streampipes/model/function/FunctionId.java
index 5a7b2f54b..7c2fdc18d 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/function/FunctionId.java
@@ -15,15 +15,30 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.commons.constants;
-public class DefaultEnvValues {
+package org.apache.streampipes.model.function;
- public static final String INITIAL_ADMIN_EMAIL_DEFAULT = "admin@streampipes.apache.org";
- public static final String INITIAL_ADMIN_PW_DEFAULT = "admin";
- public static final String INITIAL_CLIENT_USER_DEFAULT = "sp-service-client";
- public static final String INITIAL_CLIENT_SECRET_DEFAULT = "my-apache-streampipes-secret-key-change-me";
- public static final boolean INSTALL_PIPELINE_ELEMENTS = true;
+public class FunctionId {
- public static final String DEFAULT_ENCRYPTION_PASSCODE = "eGgemyGBoILAu3xckoIp";
+ private String id;
+ private int version;
+
+ public static FunctionId from(String id,
+ int version) {
+ return new FunctionId(id, version);
+ }
+
+ private FunctionId(String id,
+ int version) {
+ this.id = id;
+ this.version = version;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public int getVersion() {
+ return version;
+ }
}
diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java
new file mode 100644
index 000000000..9e803512d
--- /dev/null
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContext.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.wrapper.standalone.function;
+
+import org.apache.streampipes.client.StreamPipesClient;
+import org.apache.streampipes.container.monitoring.SpMonitoringManager;
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.monitoring.SpLogEntry;
+import org.apache.streampipes.model.schema.EventSchema;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class FunctionContext {
+
+ Map<String, SpDataStream> streams;
+ StreamPipesClient client;
+ private String functionId;
+
+ public FunctionContext() {
+ this.streams = new HashMap<>();
+ }
+
+ public FunctionContext(String functionId,
+ List<SpDataStream> streams,
+ StreamPipesClient client) {
+ this();
+ streams.forEach(stream -> this.streams.put(stream.getElementId(), stream));
+ this.functionId = functionId;
+ this.client = client;
+ }
+
+ public Collection<SpDataStream> getStreams() {
+ return streams.values();
+ }
+
+ public EventSchema getSchema(String streamId) {
+ return this.streams.get(streamId).getEventSchema();
+ }
+
+ public StreamPipesClient getClient() {
+ return client;
+ }
+
+ public void log(SpLogEntry logEntry) {
+ SpMonitoringManager.INSTANCE.addErrorMessage(functionId, logEntry);
+ }
+
+}
diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java
new file mode 100644
index 000000000..a6bae1997
--- /dev/null
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/FunctionContextGenerator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.wrapper.standalone.function;
+
+import org.apache.streampipes.client.StreamPipesClient;
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.service.extensions.base.client.StreamPipesClientResolver;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class FunctionContextGenerator {
+
+ private final List<String> streamIds;
+ private final String functionId;
+
+ public FunctionContextGenerator(String functionId,
+ List<String> streamIds) {
+ this.streamIds = streamIds;
+ this.functionId = functionId;
+ }
+
+ public FunctionContext generate() {
+ var client = makeClient();
+ var streams = receiveStreams(client);
+
+ return new FunctionContext(functionId, streams, client);
+ }
+
+ private StreamPipesClient makeClient() {
+ return new StreamPipesClientResolver().makeStreamPipesClientInstance();
+ }
+
+ private List<SpDataStream> receiveStreams(StreamPipesClient client) {
+ return this.streamIds
+ .stream()
+ .map(streamId -> client.streams().get(streamId))
+ .collect(Collectors.toList());
+ }
+}
diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java
new file mode 100644
index 000000000..7df352a22
--- /dev/null
+++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/function/StreamPipesFunction.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.wrapper.standalone.function;
+
+import org.apache.streampipes.commons.constants.Envs;
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.container.declarer.IStreamPipesFunctionDeclarer;
+import org.apache.streampipes.container.monitoring.SpMonitoringManager;
+import org.apache.streampipes.container.util.GroundingDebugUtils;
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.StreamPipesErrorMessage;
+import org.apache.streampipes.model.monitoring.SpLogEntry;
+import org.apache.streampipes.model.runtime.Event;
+import org.apache.streampipes.model.runtime.EventFactory;
+import org.apache.streampipes.wrapper.routing.RawDataProcessor;
+import org.apache.streampipes.wrapper.routing.SpInputCollector;
+import org.apache.streampipes.wrapper.standalone.manager.ProtocolManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class StreamPipesFunction implements IStreamPipesFunctionDeclarer, RawDataProcessor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StreamPipesFunction.class);
+ private Map<String, SpInputCollector> inputCollectors;
+ private final Map<String, String> sourceInfoMapper;
+
+ public StreamPipesFunction() {
+ this.sourceInfoMapper = new HashMap<>();
+ }
+
+ public void invokeRuntime() {
+ FunctionContext context = new FunctionContextGenerator(
+ this.getFunctionId().getId(), this.requiredStreamIds()
+ ).generate();
+ context.getStreams().forEach(stream -> sourceInfoMapper.put(getTopic(stream), stream.getElementId()));
+ this.inputCollectors = getInputCollectors(context.getStreams());
+ LOG.info("Invoking function {}:{}", this.getFunctionId().getId(), this.getFunctionId().getVersion());
+ onServiceStarted(context);
+ registerConsumers();
+ }
+
+ public void discardRuntime() {
+ LOG.info("Discarding function {}:{}", this.getFunctionId().getId(), this.getFunctionId().getVersion());
+ onServiceStopped();
+ unregisterConsumers();
+ }
+
+ @Override
+ public void process(Map<String, Object> rawEvent, String sourceInfo) {
+ try {
+ this.onEvent(EventFactory.fromMap(rawEvent), sourceInfoMapper.get(sourceInfo));
+ increaseCounter(sourceInfo);
+ } catch (RuntimeException e) {
+ addError(e);
+ }
+ }
+
+ private String getTopic(SpDataStream stream) {
+ return stream.getEventGrounding().getTransportProtocol().getTopicDefinition().getActualTopicName();
+ }
+
+ private void increaseCounter(String sourceInfo) {
+ SpMonitoringManager.INSTANCE.increaseInCounter(
+ this.getFunctionId().getId(),
+ sourceInfo,
+ System.currentTimeMillis()
+ );
+ }
+
+ private void addError(RuntimeException e) {
+ SpMonitoringManager.INSTANCE.addErrorMessage(
+ this.getFunctionId().getId(),
+ SpLogEntry.from(System.currentTimeMillis(), StreamPipesErrorMessage.from(e)));
+ }
+
+ private Map<String, SpInputCollector> getInputCollectors(Collection<SpDataStream> streams) throws SpRuntimeException {
+ Map<String, SpInputCollector> inputCollectors = new HashMap<>();
+ for (SpDataStream is : streams) {
+ if (Envs.SP_DEBUG.exists() && Envs.SP_DEBUG.getValueAsBoolean()) {
+ GroundingDebugUtils.modifyGrounding(is.getEventGrounding());
+ }
+ inputCollectors.put(is.getElementId(), ProtocolManager.findInputCollector(is.getEventGrounding()
+ .getTransportProtocol(), is.getEventGrounding().getTransportFormats().get(0), false));
+ }
+ return inputCollectors;
+ }
+
+ private void registerConsumers() {
+ this.inputCollectors.forEach((key, is) -> {
+ is.registerConsumer(key, this);
+ is.connect();
+ });
+ }
+
+ private void unregisterConsumers() {
+ this.inputCollectors.forEach((key, is) -> {
+ is.unregisterConsumer(key);
+ is.disconnect();
+ });
+ }
+
+ public abstract void onServiceStarted(FunctionContext context);
+
+ public abstract void onEvent(Event event,
+ String streamId);
+
+ public abstract void onServiceStopped();
+
+}