You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2026/03/11 11:55:29 UTC
(streampipes) branch dev updated: feat: Integrate Camel kamelets, add Azure sinks (#4214)
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new b4520ca7e1 feat: Integrate Camel kamelets, add Azure sinks (#4214)
b4520ca7e1 is described below
commit b4520ca7e108c12eda95230631fc54f1d9c2abe5
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Wed Mar 11 12:55:24 2026 +0100
feat: Integrate Camel kamelets, add Azure sinks (#4214)
---
pom.xml | 100 +++++
.../commons/media/ImageMimeTypeDetector.java | 60 +++
.../pe/config/IPipelineElementConfiguration.java | 6 +
streampipes-extensions/pom.xml | 2 +
.../streampipes-connectors-camel-azure/pom.xml | 125 ++++++
.../azure/CamelAzureConnectorsModuleExport.java | 22 +-
.../azure/filter/AzureKameletTemplateFilter.java | 18 +-
.../streampipes-connectors-camel/pom.xml | 84 ++++
.../assets/KameletDocumentationGenerator.java | 103 +++++
.../camel/kamelet/assets/KameletIconDecoder.java | 83 ++++
.../kamelet/assets/KameletSinkAssetResolver.java | 110 +++++
.../config/KameletConfigurationExtractor.java | 90 ++++
.../config/KameletMessageMappingExtractor.java | 172 ++++++++
.../config/KameletSinkStaticPropertyProvider.java | 241 +++++++++++
.../config/KameletStaticPropertyGenerator.java | 164 ++++++++
.../discovery/KameletTemplateDiscovery.java | 467 +++++++++++++++++++++
.../camel/kamelet/factory/KameletSinkFactory.java | 71 ++++
.../kamelet/filter/KameletTemplateFilter.java | 19 +-
.../camel/kamelet/message/CamelMessage.java | 17 +-
.../kamelet/message/KameletEventMessageMapper.java | 108 +++++
.../kamelet/message/KameletHeaderMapping.java | 15 +-
.../kamelet/message/KameletMessageMapping.java | 23 +-
.../kamelet/model/KameletPropertyDefinition.java | 51 +++
.../camel/kamelet/model/KameletTemplate.java | 22 +-
.../module/AbstractCamelKameletModuleExport.java | 81 ++++
.../provider/ClasspathKameletTemplateProvider.java | 65 +++
.../kamelet/provider/KameletTemplateProvider.java | 21 +-
.../transform/KameletTransformRouteLoader.java | 178 ++++++++
.../camel/sink/ApacheCamelKameletSink.java | 270 ++++++++++++
.../sink/GeneratedApacheCamelKameletSink.java | 42 ++
.../streampipes-extensions-all-jvm/pom.xml | 20 +
.../extensions/all/jvm/AllExtensionsInit.java | 2 +
streampipes-integration-tests/pom.xml | 12 +
.../containers/AzureCosmosDbEmulatorContainer.java | 101 +++++
.../sinks/DataSinkTestConfigurator.java | 136 ++++++
.../integration/sinks/SinkTesterBase.java | 107 +++++
.../integration/sinks/TestDataSinkParameters.java | 106 +++++
.../sinks/azure/CamelAzureCosmosDbSinkTester.java | 327 +++++++++++++++
.../azure/CamelAzureSinksIntegrationTest.java | 19 +-
.../sinks/azure/CosmosDbEmulatorTrustStore.java | 155 +++++++
.../TestAzureCosmosDbApacheCamelKameletSink.java | 106 +++++
.../sinks/azure/TestAzureKameletSinkFactory.java | 51 +++
.../TestCamelAzureConnectorsModuleExport.java | 19 +-
.../smp/extractor/ExtensionsFinder.java | 3 +-
.../resource/management/secret/SecretVisitor.java | 4 +-
.../AbstractPipelineElementResource.java | 57 +--
.../extensions/connect/AdapterAssetResource.java | 8 +-
.../rest/impl/PipelineElementAsset.java | 8 +-
.../admin/ExtensionsServiceEndpointResource.java | 7 +-
.../rest/impl/connect/DescriptionResource.java | 7 +-
.../processor/DataProcessorConfiguration.java | 19 +-
.../sdk/builder/sink/DataSinkConfiguration.java | 19 +-
.../builder/stream/DataStreamConfiguration.java | 19 +-
.../service/extensions/ExtensionItemProvider.java | 7 +-
.../pipeline-element/pipeline-element.component.ts | 4 +-
.../existing-adapters.component.ts | 2 +-
.../static-free-input.component.ts | 47 ++-
57 files changed, 4033 insertions(+), 169 deletions(-)
diff --git a/pom.xml b/pom.xml
index ded50752a3..a86837b172 100644
--- a/pom.xml
+++ b/pom.xml
@@ -47,6 +47,7 @@
<bcprov-jdk18on.version>1.78.1</bcprov-jdk18on.version>
<boofcv.version>1.1.0</boofcv.version>
<classindex.version>3.9</classindex.version>
+ <camel.version>4.18.0</camel.version>
<commons-collections.version>3.2.2</commons-collections.version>
<commoms-collection4.version>4.4</commoms-collection4.version>
<commons-compress.version>1.27.1</commons-compress.version>
@@ -114,6 +115,7 @@
<!-- dependency convergence -->
<javassist.version>3.30.1-GA</javassist.version>
<woodstox.version>7.0.0</woodstox.version>
+ <hdrhistogram.version>2.2.2</hdrhistogram.version>
<istack-runtime.version>4.1.2</istack-runtime.version>
<javax.xml.bind.version>2.3.1</javax.xml.bind.version>
<jaxb-runtime.version>2.3.9</jaxb-runtime.version>
@@ -121,6 +123,7 @@
<java-websocket.version>1.6.0</java-websocket.version>
<javax-websocket-api.version>1.1</javax-websocket-api.version>
<jetbrains.version>16.0.3</jetbrains.version>
+ <snakeyaml.version>2.5</snakeyaml.version>
<maven.compiler.source>25</maven.compiler.source>
<maven.compiler.target>25</maven.compiler.target>
@@ -157,6 +160,13 @@
<type>pom</type>
</dependency>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-sdk-bom</artifactId>
+ <version>1.3.4</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
@@ -178,6 +188,86 @@
<artifactId>commons-collections</artifactId>
<version>${commons-collections.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-cosmosdb</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-eventhubs</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-servicebus</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-storage-blob</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-storage-datalake</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-storage-queue</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-files</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-jackson</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-vertx-http</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.kamelets</groupId>
+ <artifactId>camel-kamelets-catalog</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-core-engine</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-core-languages</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-bean</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-direct</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-kamelet</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-yaml-dsl</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-jexl3</artifactId>
@@ -648,6 +738,11 @@
<artifactId>polyglot</artifactId>
<version>${graalvm.js.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.hdrhistogram</groupId>
+ <artifactId>HdrHistogram</artifactId>
+ <version>${hdrhistogram.version}</version>
+ </dependency>
<dependency>
<groupId>org.influxdb</groupId>
<artifactId>influxdb-java</artifactId>
@@ -688,6 +783,11 @@
<artifactId>simple-java-mail</artifactId>
<version>${simple-java-mail.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.snakeyaml</groupId>
+ <artifactId>snakeyaml-engine</artifactId>
+ <version>${snakeyaml.version}</version>
+ </dependency>
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/media/ImageMimeTypeDetector.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/media/ImageMimeTypeDetector.java
new file mode 100644
index 0000000000..c957c43380
--- /dev/null
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/media/ImageMimeTypeDetector.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.commons.media;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+
+public final class ImageMimeTypeDetector {
+
+ private static final String DEFAULT_IMAGE_TYPE = "image/png";
+ private static final String SVG_IMAGE_TYPE = "image/svg+xml";
+
+ private ImageMimeTypeDetector() {
+ }
+
+ public static String detect(byte[] imageBytes) {
+ if (imageBytes == null || imageBytes.length == 0) {
+ return DEFAULT_IMAGE_TYPE;
+ }
+
+ if (isSvg(imageBytes)) {
+ return SVG_IMAGE_TYPE;
+ }
+
+ try {
+ String detected = URLConnection.guessContentTypeFromStream(new ByteArrayInputStream(imageBytes));
+ return detected == null ? DEFAULT_IMAGE_TYPE : detected;
+ } catch (IOException e) {
+ return DEFAULT_IMAGE_TYPE;
+ }
+ }
+
+ private static boolean isSvg(byte[] imageBytes) {
+ int length = Math.min(imageBytes.length, 512);
+ String prefix = new String(imageBytes, 0, length, StandardCharsets.UTF_8)
+ .replace("\uFEFF", "")
+ .trim();
+
+ return prefix.startsWith("<svg")
+ || prefix.startsWith("<?xml") && prefix.contains("<svg");
+ }
+}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
index e01f7adeb1..e5e79018ec 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
@@ -18,6 +18,8 @@
package org.apache.streampipes.extensions.api.pe.config;
+import org.apache.streampipes.extensions.api.assets.AssetResolver;
+import org.apache.streampipes.extensions.api.assets.DefaultAssetResolver;
import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
import org.apache.streampipes.model.base.NamedStreamPipesEntity;
@@ -29,4 +31,8 @@ public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntit
NeT getDescription();
Supplier<PeT> getSupplier();
+
+ default AssetResolver getAssetResolver() {
+ return new DefaultAssetResolver(getDescription().getAppId());
+ }
}
diff --git a/streampipes-extensions/pom.xml b/streampipes-extensions/pom.xml
index 2db1ae8a69..d8b65906d4 100644
--- a/streampipes-extensions/pom.xml
+++ b/streampipes-extensions/pom.xml
@@ -34,6 +34,7 @@
<module>streampipes-connect-adapters</module>
<module>streampipes-connect-adapters-iiot</module>
+ <module>streampipes-connectors-camel</module>
<module>streampipes-connectors-influx</module>
<module>streampipes-connectors-kafka</module>
<module>streampipes-connectors-mqtt</module>
@@ -65,6 +66,7 @@
<module>streampipes-extensions-iiot-minimal</module>
<module>streampipes-connectors-ros</module>
<module>streampipes-processors-llm-jvm</module>
+ <module>streampipes-connectors-camel-azure</module>
</modules>
<properties>
diff --git a/streampipes-extensions/streampipes-connectors-camel-azure/pom.xml b/streampipes-extensions/streampipes-connectors-camel-azure/pom.xml
new file mode 100644
index 0000000000..a55ae8f6cb
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel-azure/pom.xml
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-extensions</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>streampipes-connectors-camel-azure</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-connectors-camel</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-cosmosdb</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>msal4j-persistence-extension</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.micrometer</groupId>
+ <artifactId>micrometer-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-eventhubs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-servicebus</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-storage-blob</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-storage-blob-changefeed</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-storage-datalake</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-storage-queue</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-files</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-jackson</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-vertx-http</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.kamelets</groupId>
+ <artifactId>camel-kamelets-catalog</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>unpack-azure-kamelets</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.camel.kamelets</groupId>
+ <artifactId>camel-kamelets</artifactId>
+ <version>${camel.version}</version>
+ <includes>kamelets/azure-cosmosdb-sink.kamelet.yaml,kamelets/azure-eventhubs-sink.kamelet.yaml,kamelets/azure-functions-sink.kamelet.yaml,kamelets/azure-servicebus-sink.kamelet.yaml,kamelets/azure-storage-blob-append-sink.kamelet.yaml,kamelets/azure-storage-blob-sink.kamelet.yaml,kamelets/azure-storage-datalake-sink.kamelet.yaml,kamelets/azure-storage-files-sink.kamelet.yaml,kamelets/azure-storage-queue-sink.kamelet.yaml</includes>
+ <outputDirectory>${project.build.outputDirectory}</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-extensions/streampipes-connectors-camel-azure/src/main/java/org/apache/streampipes/extensions/connectors/camel/azure/CamelAzureConnectorsModuleExport.java
similarity index 54%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-extensions/streampipes-connectors-camel-azure/src/main/java/org/apache/streampipes/extensions/connectors/camel/azure/CamelAzureConnectorsModuleExport.java
index e01f7adeb1..62ae4c234c 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-extensions/streampipes-connectors-camel-azure/src/main/java/org/apache/streampipes/extensions/connectors/camel/azure/CamelAzureConnectorsModuleExport.java
@@ -16,17 +16,21 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.extensions.connectors.camel.azure;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.extensions.connectors.camel.azure.filter.AzureKameletTemplateFilter;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.filter.KameletTemplateFilter;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.module.AbstractCamelKameletModuleExport;
-import java.util.function.Supplier;
+public class CamelAzureConnectorsModuleExport extends AbstractCamelKameletModuleExport {
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
+ @Override
+ protected String getAppIdPrefix() {
+ return "org.apache.streampipes.sinks.forward.jvm.camel";
+ }
- NeT getDescription();
-
- Supplier<PeT> getSupplier();
+ @Override
+ protected KameletTemplateFilter getTemplateFilter() {
+ return new AzureKameletTemplateFilter();
+ }
}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-extensions/streampipes-connectors-camel-azure/src/main/java/org/apache/streampipes/extensions/connectors/camel/azure/filter/AzureKameletTemplateFilter.java
similarity index 62%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-extensions/streampipes-connectors-camel-azure/src/main/java/org/apache/streampipes/extensions/connectors/camel/azure/filter/AzureKameletTemplateFilter.java
index e01f7adeb1..5e2dab21bd 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-extensions/streampipes-connectors-camel-azure/src/main/java/org/apache/streampipes/extensions/connectors/camel/azure/filter/AzureKameletTemplateFilter.java
@@ -16,17 +16,15 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.extensions.connectors.camel.azure.filter;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.filter.KameletTemplateFilter;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
-import java.util.function.Supplier;
+public class AzureKameletTemplateFilter implements KameletTemplateFilter {
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
-
- NeT getDescription();
-
- Supplier<PeT> getSupplier();
+ @Override
+ public boolean test(KameletTemplate template) {
+ return template.name().startsWith("azure-") && template.name().endsWith("-sink");
+ }
}
diff --git a/streampipes-extensions/streampipes-connectors-camel/pom.xml b/streampipes-extensions/streampipes-connectors-camel/pom.xml
new file mode 100644
index 0000000000..4aa99b958f
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-extensions</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>streampipes-connectors-camel</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-extensions-api</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-extensions-management</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-core-engine</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-core-languages</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-bean</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-direct</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-kamelet</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-yaml-dsl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.snakeyaml</groupId>
+ <artifactId>snakeyaml-engine</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hdrhistogram</groupId>
+ <artifactId>HdrHistogram</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/assets/KameletDocumentationGenerator.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/assets/KameletDocumentationGenerator.java
new file mode 100644
index 0000000000..a320f9952d
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/assets/KameletDocumentationGenerator.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.assets;
+
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletPropertyDefinition;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class KameletDocumentationGenerator {
+
+ public byte[] generateDocumentation(KameletTemplate template) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("# ").append(template.displayName()).append("\n\n");
+
+ if (!template.description().isBlank()) {
+ builder.append(template.description()).append("\n\n");
+ }
+
+ builder.append("- Kamelet name: `").append(template.name()).append("`\n");
+ builder.append("- Endpoint: `").append(template.endpointBaseUri()).append("`\n");
+ if (!template.group().isBlank()) {
+ builder.append("- Group: `").append(template.group()).append("`\n");
+ }
+
+ if (template.properties().isEmpty()) {
+ builder.append("\n## Parameters\n\n");
+ builder.append("This Kamelet exposes no configuration parameters.\n");
+ } else {
+ appendParameterSection(builder, "Required Parameters", template.properties().stream()
+ .filter(KameletPropertyDefinition::required)
+ .toList());
+ appendParameterSection(builder, "Optional Parameters", template.properties().stream()
+ .filter(property -> !property.required())
+ .toList());
+ }
+
+ builder.append("\n## YAML Spec\n\n");
+ builder.append("```yaml\n");
+ builder.append(template.yamlSpec());
+ builder.append("\n```\n");
+
+ return builder.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ private void appendParameterSection(StringBuilder builder,
+ String title,
+ List<KameletPropertyDefinition> properties) {
+ if (properties.isEmpty()) {
+ return;
+ }
+
+ builder.append("\n## ").append(title).append("\n\n");
+ for (KameletPropertyDefinition property : properties) {
+ builder.append("- `").append(property.name()).append("`");
+ builder.append(" (`").append(property.datatype().name()).append("`, ");
+ builder.append(property.inputType().name()).append(")");
+
+ if (property.defaultValue() != null && !property.defaultValue().isBlank()) {
+ builder.append(", default `").append(escapeInline(property.defaultValue())).append("`");
+ }
+
+ if (!property.allowedValues().isEmpty()) {
+ builder.append(", allowed: ");
+ builder.append(property.allowedValues().stream()
+ .map(this::escapeInline)
+ .map(value -> "`" + value + "`")
+ .collect(Collectors.joining(", ")));
+ }
+
+ builder.append("\n");
+ builder.append(" ");
+ builder.append(escapeText(property.description().isBlank() ? property.displayLabel() : property.description()));
+ builder.append("\n");
+ }
+ }
+
+ private String escapeInline(String value) {
+ return value == null ? "" : value.replace("`", "\\`").replace("\n", " ");
+ }
+
+ private String escapeText(String value) {
+ return value == null ? "" : value.replace("\n", " ").trim();
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/assets/KameletIconDecoder.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/assets/KameletIconDecoder.java
new file mode 100644
index 0000000000..f084df19b7
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/assets/KameletIconDecoder.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.assets;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Base64;
+import java.util.Locale;
+
+public class KameletIconDecoder {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KameletIconDecoder.class);
+
+ public byte[] decode(String iconValue) {
+ if (iconValue == null || iconValue.isBlank()) {
+ return null;
+ }
+
+ try {
+ DecodedIcon decodedIcon = decodeIcon(iconValue);
+ if (decodedIcon == null || decodedIcon.bytes.length == 0) {
+ return null;
+ }
+ return decodedIcon.bytes;
+ } catch (Exception e) {
+ LOG.debug("Skipping invalid embedded Kamelet icon", e);
+ return null;
+ }
+ }
+
+ private DecodedIcon decodeIcon(String iconValue) {
+ String trimmed = iconValue.trim();
+ if (trimmed.startsWith("data:")) {
+ return decodeDataUri(trimmed);
+ }
+
+ byte[] bytes = Base64.getMimeDecoder().decode(trimmed);
+ return new DecodedIcon("image/png", bytes);
+ }
+
+ private DecodedIcon decodeDataUri(String dataUri) {
+ int commaIndex = dataUri.indexOf(',');
+ if (commaIndex < 0) {
+ throw new IllegalArgumentException("Invalid data URI");
+ }
+
+ String metadata = dataUri.substring(5, commaIndex);
+ String payload = dataUri.substring(commaIndex + 1);
+ String mediaType = metadata.contains(";")
+ ? metadata.substring(0, metadata.indexOf(';'))
+ : metadata;
+
+ if (!metadata.contains(";base64")) {
+ throw new IllegalArgumentException("Only base64-encoded icon data URIs are supported");
+ }
+
+ if (mediaType.isBlank()) {
+ mediaType = "image/png";
+ }
+
+ return new DecodedIcon(mediaType.toLowerCase(Locale.ROOT), Base64.getMimeDecoder().decode(payload));
+ }
+
+ private record DecodedIcon(String mediaType, byte[] bytes) {
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/assets/KameletSinkAssetResolver.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/assets/KameletSinkAssetResolver.java
new file mode 100644
index 0000000000..46ee02467d
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/assets/KameletSinkAssetResolver.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.assets;
+
+import org.apache.streampipes.commons.constants.GlobalStreamPipesConstants;
+import org.apache.streampipes.extensions.api.assets.AssetResolver;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.provider.KameletTemplateProvider;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
+
+public class KameletSinkAssetResolver implements AssetResolver {
+
+ private final String appId;
+ private final String templateName;
+ private final String assetRoot;
+ private final KameletTemplateProvider templateProvider;
+ private final KameletDocumentationGenerator documentationGenerator;
+
+ public KameletSinkAssetResolver(String appId,
+ String templateName,
+ String assetRoot,
+ KameletTemplateProvider templateProvider) {
+ this.appId = appId;
+ this.templateName = templateName;
+ this.assetRoot = assetRoot;
+ this.templateProvider = templateProvider;
+ this.documentationGenerator = new KameletDocumentationGenerator();
+ }
+
+ @Override
+ public byte[] getAsset(ClassLoader classLoader,
+ String assetName) throws IOException {
+ KameletTemplate template = templateProvider.requireTemplate(templateName);
+
+ if (GlobalStreamPipesConstants.STD_DOCUMENTATION_NAME.equals(assetName)) {
+ return documentationGenerator.generateDocumentation(template);
+ }
+
+ if (GlobalStreamPipesConstants.STD_ICON_NAME.equals(assetName)) {
+ InputStream iconStream = getResourceStream(classLoader, makeAssetPath(template.name(), assetName));
+ if (iconStream == null) {
+ iconStream = getResourceStream(classLoader, makeAssetPath(template.name(), "icon.svg"));
+ }
+ if (iconStream != null) {
+ return iconStream.readAllBytes();
+ }
+
+ if (template.embeddedIcon() != null && template.embeddedIcon().length > 0) {
+ return template.embeddedIcon();
+ }
+
+ throw new IOException(String.format("Could not read icon for template %s", template.name()));
+ }
+
+ InputStream stream = getResourceStream(classLoader, makeAssetPath(template.name(), assetName));
+ if (stream == null) {
+ throw new IOException(String.format("Could not read asset %s for template %s", assetName, template.name()));
+ }
+
+ return stream.readAllBytes();
+ }
+
+ @Override
+ public Properties getLocale(ClassLoader classLoader,
+ String localeName) throws IOException {
+ KameletTemplate template = templateProvider.requireTemplate(templateName);
+ Properties properties = new Properties();
+
+ InputStream localeStream = getResourceStream(classLoader, makeAssetPath(template.name(), localeName));
+ if (localeStream != null) {
+ properties.load(new InputStreamReader(localeStream, StandardCharsets.UTF_8));
+ }
+
+ properties.put(appId + ".title", template.displayName());
+ properties.put(appId + ".description", template.description());
+
+ return properties;
+ }
+
+ private String makeAssetPath(String templateName,
+ String assetName) {
+ return assetRoot + "/" + templateName + "/" + assetName;
+ }
+
+ private InputStream getResourceStream(ClassLoader classLoader,
+ String path) {
+ return classLoader.getResourceAsStream(path);
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletConfigurationExtractor.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletConfigurationExtractor.java
new file mode 100644
index 0000000000..4a2b560cfe
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletConfigurationExtractor.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.config;
+
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletPropertyDefinition;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty;
+import org.apache.streampipes.model.staticproperty.OneOfStaticProperty;
+import org.apache.streampipes.model.staticproperty.Option;
+import org.apache.streampipes.model.staticproperty.RuntimeResolvableGroupStaticProperty;
+import org.apache.streampipes.model.staticproperty.SecretStaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class KameletConfigurationExtractor {
+
+ public Map<String, Object> buildParams(KameletTemplate template,
+ RuntimeResolvableGroupStaticProperty parameterGroup) {
+ LinkedHashMap<String, Object> parameters = new LinkedHashMap<>();
+
+ for (KameletPropertyDefinition definition : template.properties()) {
+ StaticProperty property = findProperty(parameterGroup, definition.staticPropertyInternalName(template.name()));
+ if (property == null) {
+ if (definition.required()) {
+ throw new SpRuntimeException("Missing required Kamelet parameter: " + definition.name());
+ }
+ continue;
+ }
+
+ String value = extractValue(definition, property);
+ if (value == null || value.isBlank()) {
+ if (definition.required()) {
+ throw new SpRuntimeException("Missing required Kamelet parameter: " + definition.name());
+ }
+ continue;
+ }
+
+ parameters.put(definition.name(), value);
+ }
+
+ return parameters;
+ }
+
+ private StaticProperty findProperty(RuntimeResolvableGroupStaticProperty parameterGroup,
+ String internalName) {
+ return parameterGroup.getStaticProperties()
+ .stream()
+ .filter(property -> Objects.equals(property.getInternalName(), internalName))
+ .findFirst()
+ .orElse(null);
+ }
+
+ private String extractValue(KameletPropertyDefinition definition,
+ StaticProperty property) {
+ return switch (definition.inputType()) {
+ case TEXT -> ((FreeTextStaticProperty) property).getValue();
+ case SECRET -> ((SecretStaticProperty) property).getValue();
+ case ONE_OF -> extractSelectedOption((OneOfStaticProperty) property);
+ };
+ }
+
+ private String extractSelectedOption(OneOfStaticProperty property) {
+ return property.getOptions()
+ .stream()
+ .filter(Option::isSelected)
+ .findFirst()
+ .map(option -> option.getInternalName() == null ? option.getName() : option.getInternalName())
+ .orElse("");
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletMessageMappingExtractor.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletMessageMappingExtractor.java
new file mode 100644
index 0000000000..3c218bde15
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletMessageMappingExtractor.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.config;
+
+import org.apache.streampipes.extensions.api.extractor.IParameterExtractor;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.message.KameletHeaderMapping;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.message.KameletMessageMapping;
+import org.apache.streampipes.model.staticproperty.CodeInputStaticProperty;
+import org.apache.streampipes.model.staticproperty.CollectionStaticProperty;
+import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty;
+import org.apache.streampipes.model.staticproperty.MappingPropertyUnary;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticPropertyAlternative;
+import org.apache.streampipes.model.staticproperty.StaticPropertyAlternatives;
+import org.apache.streampipes.model.staticproperty.StaticPropertyGroup;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+public class KameletMessageMappingExtractor {
+
+ public KameletMessageMapping extract(IParameterExtractor extractor) {
+ KameletMessageMapping.PayloadMode payloadMode = KameletMessageMapping.PayloadMode.EVENT_MAP;
+ String bodyFieldSelector = null;
+
+ StaticPropertyAlternatives payloadAlternatives = extractor.getStaticPropertyByName(
+ KameletSinkStaticPropertyProvider.PAYLOAD_ALTERNATIVES_KEY,
+ StaticPropertyAlternatives.class
+ );
+
+ if (payloadAlternatives != null) {
+ Optional<StaticPropertyAlternative> selectedAlternative = getSelectedAlternative(payloadAlternatives);
+ if (selectedAlternative.isPresent()) {
+ String internalName = selectedAlternative.get().getInternalName();
+ if (KameletSinkStaticPropertyProvider.PAYLOAD_EVENT_JSON_KEY.equals(internalName)) {
+ payloadMode = KameletMessageMapping.PayloadMode.EVENT_JSON;
+ } else if (KameletSinkStaticPropertyProvider.PAYLOAD_MAPPED_FIELD_KEY.equals(internalName)) {
+ payloadMode = KameletMessageMapping.PayloadMode.MAPPED_FIELD;
+ bodyFieldSelector = extractMappedBodyField(selectedAlternative.get().getStaticProperty());
+ }
+ }
+ }
+
+ return new KameletMessageMapping(
+ payloadMode,
+ bodyFieldSelector,
+ extractHeaderMappings(extractor),
+ extractTransformSteps(extractor)
+ );
+ }
+
+ private String extractMappedBodyField(StaticProperty payloadProperty) {
+ if (!(payloadProperty instanceof StaticPropertyGroup group)) {
+ return null;
+ }
+
+ StaticProperty property = findGroupMember(group, KameletSinkStaticPropertyProvider.PAYLOAD_FIELD_MAPPING_KEY);
+ if (property instanceof MappingPropertyUnary mappingProperty) {
+ return mappingProperty.getSelectedProperty();
+ }
+
+ return null;
+ }
+
+ private List<KameletHeaderMapping> extractHeaderMappings(IParameterExtractor extractor) {
+ CollectionStaticProperty collection = extractor.getStaticPropertyByName(
+ KameletSinkStaticPropertyProvider.HEADER_MAPPINGS_KEY,
+ CollectionStaticProperty.class
+ );
+
+ if (collection == null || collection.getMembers() == null) {
+ return List.of();
+ }
+
+ List<KameletHeaderMapping> mappings = new ArrayList<>();
+
+ for (StaticProperty member : collection.getMembers()) {
+ if (!(member instanceof StaticPropertyGroup group)) {
+ continue;
+ }
+
+ String headerName = extractHeaderName(group);
+ String eventFieldSelector = extractHeaderFieldSelector(group);
+
+ if (headerName != null && !headerName.isBlank() && eventFieldSelector != null && !eventFieldSelector.isBlank()) {
+ mappings.add(new KameletHeaderMapping(headerName, eventFieldSelector));
+ }
+ }
+
+ return mappings;
+ }
+
+ private String extractHeaderName(StaticPropertyGroup group) {
+ StaticProperty property = findGroupMember(group, KameletSinkStaticPropertyProvider.HEADER_NAME_KEY);
+ if (property instanceof FreeTextStaticProperty freeTextStaticProperty) {
+ return freeTextStaticProperty.getValue();
+ }
+
+ return null;
+ }
+
+ private String extractHeaderFieldSelector(StaticPropertyGroup group) {
+ StaticProperty property = findGroupMember(group, KameletSinkStaticPropertyProvider.HEADER_FIELD_MAPPING_KEY);
+ if (property instanceof MappingPropertyUnary mappingProperty) {
+ return mappingProperty.getSelectedProperty();
+ }
+
+ return null;
+ }
+
+ private String extractTransformSteps(IParameterExtractor extractor) {
+ StaticPropertyAlternatives alternatives = extractor.getStaticPropertyByName(
+ KameletSinkStaticPropertyProvider.ADVANCED_TRANSFORM_KEY,
+ StaticPropertyAlternatives.class
+ );
+
+ if (alternatives == null) {
+ return null;
+ }
+
+ Optional<StaticPropertyAlternative> selectedAlternative = getSelectedAlternative(alternatives);
+ if (selectedAlternative.isEmpty()
+ || !KameletSinkStaticPropertyProvider.ADVANCED_TRANSFORM_ENABLED_KEY.equals(
+ selectedAlternative.get().getInternalName())) {
+ return null;
+ }
+
+ if (!(selectedAlternative.get().getStaticProperty() instanceof StaticPropertyGroup group)) {
+ return null;
+ }
+
+ StaticProperty property = findGroupMember(group, KameletSinkStaticPropertyProvider.ADVANCED_TRANSFORM_STEPS_KEY);
+ if (property instanceof CodeInputStaticProperty codeInputStaticProperty) {
+ return codeInputStaticProperty.getValue();
+ }
+
+ return null;
+ }
+
+ private Optional<StaticPropertyAlternative> getSelectedAlternative(StaticPropertyAlternatives alternatives) {
+ return alternatives.getAlternatives()
+ .stream()
+ .filter(StaticPropertyAlternative::getSelected)
+ .findFirst();
+ }
+
+ private StaticProperty findGroupMember(StaticPropertyGroup group,
+ String internalName) {
+ return group.getStaticProperties()
+ .stream()
+ .filter(property -> internalName.equals(property.getInternalName()))
+ .findFirst()
+ .orElse(null);
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletSinkStaticPropertyProvider.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletSinkStaticPropertyProvider.java
new file mode 100644
index 0000000000..7c160d625b
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletSinkStaticPropertyProvider.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.config;
+
+import org.apache.streampipes.model.schema.PropertyScope;
+import org.apache.streampipes.model.staticproperty.CodeInputStaticProperty;
+import org.apache.streampipes.model.staticproperty.CollectionStaticProperty;
+import org.apache.streampipes.model.staticproperty.MappingPropertyUnary;
+import org.apache.streampipes.model.staticproperty.RuntimeResolvableGroupStaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticPropertyAlternative;
+import org.apache.streampipes.model.staticproperty.StaticPropertyAlternatives;
+import org.apache.streampipes.model.staticproperty.StaticPropertyGroup;
+import org.apache.streampipes.sdk.StaticProperties;
+import org.apache.streampipes.sdk.helpers.CodeLanguage;
+import org.apache.streampipes.sdk.helpers.Labels;
+
+import java.util.List;
+
+public class KameletSinkStaticPropertyProvider {
+
+ public static final String KAMELET_PARAMETERS_GROUP_KEY = "camel-kamelet-parameters-group";
+
+ public static final String MESSAGE_MAPPING_GROUP_KEY = "camel-message-mapping-group";
+ public static final String PAYLOAD_GROUP_KEY = "camel-payload-group";
+ public static final String PAYLOAD_ALTERNATIVES_KEY = "camel-payload-mode";
+ public static final String PAYLOAD_EVENT_MAP_KEY = "camel-payload-event-map";
+ public static final String PAYLOAD_EVENT_JSON_KEY = "camel-payload-event-json";
+ public static final String PAYLOAD_MAPPED_FIELD_KEY = "camel-payload-mapped-field";
+ public static final String PAYLOAD_FIELD_MAPPING_KEY = "camel-payload-field";
+ public static final String HEADER_MAPPINGS_KEY = "camel-header-mappings";
+ public static final String HEADER_MAPPING_GROUP_KEY = "camel-header-mapping-group";
+ public static final String HEADER_NAME_KEY = "camel-header-name";
+ public static final String HEADER_FIELD_MAPPING_KEY = "camel-header-field";
+
+ public static final String ADVANCED_TRANSFORM_KEY = "camel-advanced-transform";
+ public static final String ADVANCED_TRANSFORM_DISABLED_KEY = "camel-advanced-transform-disabled";
+ public static final String ADVANCED_TRANSFORM_ENABLED_KEY = "camel-advanced-transform-enabled";
+ public static final String ADVANCED_TRANSFORM_GROUP_KEY = "camel-advanced-transform-group";
+ public static final String ADVANCED_TRANSFORM_STEPS_KEY = "camel-advanced-transform-steps";
+
+ public RuntimeResolvableGroupStaticProperty createKameletParameterGroup() {
+ var group = StaticProperties.runtimeResolvableGroupStaticProperty(
+ Labels.from(
+ KAMELET_PARAMETERS_GROUP_KEY,
+ "Parameters",
+ "Configure the Apache Camel Kamelet parameters."
+ ),
+ List.of()
+ );
+ group.setHorizontalRendering(false);
+ return group;
+ }
+
+ public StaticPropertyGroup createMessageMappingGroup() {
+ StaticPropertyGroup group = StaticProperties.group(
+ Labels.from(
+ MESSAGE_MAPPING_GROUP_KEY,
+ "Message Mapping",
+ "Control how the StreamPipes event is exposed to Camel before the Kamelet is invoked."
+ ),
+ false,
+ createPayloadAlternatives(),
+ createHeaderMappingsCollection()
+ );
+ group.setHorizontalRendering(false);
+ return group;
+ }
+
+ public StaticPropertyAlternatives createAdvancedTransformAlternatives() {
+ return StaticProperties.alternatives(
+ Labels.from(
+ ADVANCED_TRANSFORM_KEY,
+ "Advanced Transform",
+ "Optionally apply custom Camel YAML steps before the Kamelet sink endpoint."
+ ),
+ List.of(
+ alternative(
+ ADVANCED_TRANSFORM_DISABLED_KEY,
+ "Disabled",
+ "Use only the structured Kamelet mapping configuration.",
+ true,
+ emptyGroup("camel-advanced-transform-disabled-group")
+ ),
+ alternative(
+ ADVANCED_TRANSFORM_ENABLED_KEY,
+ "Custom YAML Steps",
+ "Apply additional Camel YAML steps before forwarding the message to the Kamelet.",
+ false,
+ advancedTransformGroup()
+ )
+ )
+ );
+ }
+
+ private StaticPropertyAlternatives createPayloadAlternatives() {
+ return StaticProperties.alternatives(
+ Labels.from(
+ PAYLOAD_ALTERNATIVES_KEY,
+ "Payload",
+ "Choose how the input event is exposed as the Camel message body."
+ ),
+ List.of(
+ alternative(
+ PAYLOAD_EVENT_MAP_KEY,
+ "Full Event Map",
+ "Send the entire input event as a map body.",
+ true,
+ emptyGroup(PAYLOAD_EVENT_MAP_KEY + "-group")
+ ),
+ alternative(
+ PAYLOAD_EVENT_JSON_KEY,
+ "Full Event JSON",
+ "Send the entire input event as a JSON string body.",
+ false,
+ emptyGroup(PAYLOAD_EVENT_JSON_KEY + "-group")
+ ),
+ alternative(
+ PAYLOAD_MAPPED_FIELD_KEY,
+ "Mapped Field",
+ "Use one selected input field as the Camel message body.",
+ false,
+ mappedFieldPayloadGroup()
+ )
+ )
+ );
+ }
+
+ private StaticPropertyGroup mappedFieldPayloadGroup() {
+ MappingPropertyUnary bodyField = createUnaryMappingWithoutRequirement(
+ PAYLOAD_FIELD_MAPPING_KEY,
+ "Body Field",
+ "Select the input event field that should become the Camel message body."
+ );
+
+ StaticPropertyGroup group = StaticProperties.group(
+ Labels.from(PAYLOAD_GROUP_KEY, "", ""),
+ false,
+ bodyField
+ );
+ group.setHorizontalRendering(false);
+ return group;
+ }
+
+ private CollectionStaticProperty createHeaderMappingsCollection() {
+ StaticPropertyGroup headerMappingTemplate = StaticProperties.group(
+ Labels.from(
+ HEADER_MAPPING_GROUP_KEY,
+ "Header Mapping",
+ "Map one input field to one Camel header."
+ ),
+ false,
+ StaticProperties.stringFreeTextProperty(
+ Labels.from(
+ HEADER_NAME_KEY,
+ "Header Name",
+ "The Camel header name to set."
+ )
+ ),
+ createUnaryMappingWithoutRequirement(
+ HEADER_FIELD_MAPPING_KEY,
+ "Event Field",
+ "Select the input event field to map to this Camel header."
+ )
+ );
+ headerMappingTemplate.setHorizontalRendering(false);
+
+ return new CollectionStaticProperty(
+ HEADER_MAPPINGS_KEY,
+ "Header Mappings",
+ "Optionally project selected input fields into Camel headers.",
+ headerMappingTemplate
+ );
+ }
+
+ private StaticPropertyGroup advancedTransformGroup() {
+ CodeInputStaticProperty codeBlock = StaticProperties.codeStaticProperty(
+ Labels.from(
+ ADVANCED_TRANSFORM_STEPS_KEY,
+ "Transform Steps (YAML)",
+ "Provide a Camel YAML steps fragment that runs before the message is sent to the Kamelet."
+ ),
+ CodeLanguage.None,
+ """
+ - setHeader:
+ name: example
+ simple: "${body[myField]}"
+ """
+ );
+ codeBlock.setLanguage("yaml");
+
+ StaticPropertyGroup group = StaticProperties.group(
+ Labels.from(ADVANCED_TRANSFORM_GROUP_KEY, "", ""),
+ false,
+ codeBlock
+ );
+ group.setHorizontalRendering(false);
+ return group;
+ }
+
+ private StaticPropertyAlternative alternative(String internalName,
+ String label,
+ String description,
+ boolean selected,
+ StaticProperty property) {
+ StaticPropertyAlternative alternative = new StaticPropertyAlternative(internalName, label, description);
+ alternative.setSelected(selected);
+ alternative.setStaticProperty(property);
+ return alternative;
+ }
+
+ private StaticPropertyGroup emptyGroup(String internalName) {
+ StaticPropertyGroup group = StaticProperties.group(Labels.from(internalName, "", ""), false);
+ group.setHorizontalRendering(false);
+ return group;
+ }
+
+ private MappingPropertyUnary createUnaryMappingWithoutRequirement(String internalName,
+ String label,
+ String description) {
+ MappingPropertyUnary mappingProperty = new MappingPropertyUnary(internalName, label, description);
+ mappingProperty.setPropertyScope(PropertyScope.NONE.name());
+ return mappingProperty;
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletStaticPropertyGenerator.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletStaticPropertyGenerator.java
new file mode 100644
index 0000000000..eb3e9e5627
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/config/KameletStaticPropertyGenerator.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.config;
+
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletPropertyDefinition;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty;
+import org.apache.streampipes.model.staticproperty.OneOfStaticProperty;
+import org.apache.streampipes.model.staticproperty.Option;
+import org.apache.streampipes.model.staticproperty.RuntimeResolvableGroupStaticProperty;
+import org.apache.streampipes.model.staticproperty.SecretStaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+import org.apache.streampipes.sdk.StaticProperties;
+import org.apache.streampipes.sdk.helpers.Labels;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+public class KameletStaticPropertyGenerator {
+
+ private static final String OPTIONAL_EMPTY_OPTION_LABEL = "Not set";
+
+ public RuntimeResolvableGroupStaticProperty configureParameterGroup(RuntimeResolvableGroupStaticProperty group,
+ KameletTemplate template) {
+ group.setShowLabel(false);
+ group.setHorizontalRendering(false);
+ List<StaticProperty> existingProperties = Objects.nonNull(group.getStaticProperties()) ? group.getStaticProperties() : new ArrayList<>();
+ group.setStaticProperties(template.properties().stream().map(property -> toStaticProperty(template, property, existingProperties)).toList());
+ return group;
+ }
+
+ private StaticProperty toStaticProperty(KameletTemplate template,
+ KameletPropertyDefinition propertyDefinition,
+ List<StaticProperty> existingProperties) {
+ String internalName = propertyDefinition.staticPropertyInternalName(template.name());
+ var label = Labels.from(
+ internalName,
+ propertyDefinition.displayLabel(),
+ propertyDefinition.description()
+ );
+
+ return switch (propertyDefinition.inputType()) {
+ case TEXT -> makeTextProperty(label, propertyDefinition, getExisting(internalName, existingProperties, FreeTextStaticProperty.class));
+ case SECRET -> makeSecretProperty(label, propertyDefinition, getExisting(internalName, existingProperties, SecretStaticProperty.class));
+ case ONE_OF -> makeOneOfProperty(label, propertyDefinition, getExisting(internalName, existingProperties, OneOfStaticProperty.class));
+ };
+ }
+
+ private StaticProperty makeTextProperty(org.apache.streampipes.sdk.helpers.Label label,
+ KameletPropertyDefinition definition,
+ FreeTextStaticProperty existing) {
+ FreeTextStaticProperty property = StaticProperties.freeTextProperty(label, definition.datatype());
+
+ property.setOptional(!definition.required());
+ if (Objects.nonNull(existing)) {
+ property.setValue(existing.getValue());
+ } else if (definition.defaultValue() != null) {
+ property.setValue(definition.defaultValue());
+ }
+ return property;
+ }
+
+ private StaticProperty makeSecretProperty(org.apache.streampipes.sdk.helpers.Label label,
+ KameletPropertyDefinition definition,
+ SecretStaticProperty existing) {
+ SecretStaticProperty property = StaticProperties.secretValue(label);
+ property.setOptional(!definition.required());
+ if (Objects.nonNull(existing)) {
+ property.setValue(existing.getValue());
+ } else if (definition.defaultValue() != null) {
+ property.setValue(definition.defaultValue());
+ }
+ return property;
+ }
+
+ private <T extends StaticProperty> T getExisting(String internalName,
+ List<StaticProperty> existingProperties,
+ Class<T> targetClass) {
+ return existingProperties
+ .stream()
+ .filter(sp -> sp.getInternalName().equals(internalName))
+ .filter(targetClass::isInstance)
+ .map(targetClass::cast)
+ .findFirst()
+ .orElse(null);
+ }
+
+ private StaticProperty makeOneOfProperty(org.apache.streampipes.sdk.helpers.Label label,
+ KameletPropertyDefinition definition,
+ OneOfStaticProperty existing) {
+ List<Option> options = new ArrayList<>();
+
+ if (!definition.required()) {
+ Option emptyOption = new Option(OPTIONAL_EMPTY_OPTION_LABEL, "");
+ emptyOption.setSelected(definition.defaultValue() == null);
+ options.add(emptyOption);
+ }
+
+ for (String value : definition.allowedValues()) {
+ Option option = new Option(value, value);
+ option.setSelected(value.equals(definition.defaultValue()));
+ options.add(option);
+ }
+
+ Optional<String> existingSelection = getSelectedOptionInternalName(existing);
+ if (existingSelection.isPresent()) {
+ boolean matched = false;
+ for (Option option : options) {
+ boolean selected = Objects.equals(option.getInternalName(), existingSelection.get());
+ option.setSelected(selected);
+ matched |= selected;
+ }
+
+ if (!matched) {
+ applyDefaultSelection(options);
+ }
+ } else if (options.stream().noneMatch(Option::isSelected) && !options.isEmpty()) {
+ applyDefaultSelection(options);
+ }
+
+ OneOfStaticProperty property = StaticProperties.singleValueSelection(label, options);
+ property.setOptional(!definition.required());
+ return property;
+ }
+
+ private Optional<String> getSelectedOptionInternalName(OneOfStaticProperty existing) {
+ if (Objects.isNull(existing) || Objects.isNull(existing.getOptions())) {
+ return Optional.empty();
+ }
+
+ return existing.getOptions()
+ .stream()
+ .filter(Option::isSelected)
+ .map(option -> option.getInternalName() == null ? option.getName() : option.getInternalName())
+ .findFirst();
+ }
+
+ private void applyDefaultSelection(List<Option> options) {
+ if (options.isEmpty()) {
+ return;
+ }
+
+ options.forEach(option -> option.setSelected(false));
+ options.get(0).setSelected(true);
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/discovery/KameletTemplateDiscovery.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/discovery/KameletTemplateDiscovery.java
new file mode 100644
index 0000000000..6af774a6a9
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/discovery/KameletTemplateDiscovery.java
@@ -0,0 +1,467 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.discovery;
+
+import org.apache.streampipes.commons.exceptions.SpConfigurationException;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.assets.KameletIconDecoder;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.filter.KameletTemplateFilter;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletPropertyDefinition;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+import org.apache.streampipes.sdk.utils.Datatypes;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.snakeyaml.engine.v2.api.Load;
+import org.snakeyaml.engine.v2.api.LoadSettings;
+
+import java.net.JarURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.stream.Collectors;
+
+public class KameletTemplateDiscovery {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KameletTemplateDiscovery.class);
+ private static final Load YAML_LOADER = new Load(LoadSettings.builder().build());
+
+ private final KameletIconDecoder iconDecoder;
+
+ public KameletTemplateDiscovery() {
+ this.iconDecoder = new KameletIconDecoder();
+ }
+
+ public Map<String, KameletTemplate> discoverTemplates() throws SpConfigurationException {
+ return discoverTemplates(KameletTemplateFilter.includeAll());
+ }
+
+ public Map<String, KameletTemplate> discoverTemplates(KameletTemplateFilter filter) throws SpConfigurationException {
+ try {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ Enumeration<URL> resources = classLoader.getResources("kamelets");
+ List<KameletTemplate> templates = new ArrayList<>();
+ Set<String> seenNames = new HashSet<>();
+
+ while (resources.hasMoreElements()) {
+ URL url = resources.nextElement();
+ if ("file".equals(url.getProtocol())) {
+ collectFromFileDirectory(url, templates, seenNames);
+ } else if ("jar".equals(url.getProtocol())) {
+ collectFromJar(url, templates, seenNames);
+ } else {
+ LOG.debug("Skipping unsupported kamelets resource URL protocol {} ({})", url.getProtocol(), url);
+ }
+ }
+
+ return templates.stream()
+ .filter(filter::test)
+ .sorted(Comparator.comparing(KameletTemplate::displayName))
+ .collect(Collectors.toMap(
+ KameletTemplate::name,
+ template -> template,
+ (first, second) -> first,
+ LinkedHashMap::new
+ ));
+ } catch (Exception e) {
+ throw new SpConfigurationException("Could not discover registered Kamelet templates", e);
+ }
+ }
+
+ private void collectFromFileDirectory(URL directoryUrl,
+ List<KameletTemplate> templates,
+ Set<String> seenNames) throws Exception {
+ Path directory = Path.of(directoryUrl.toURI());
+ if (!Files.isDirectory(directory)) {
+ return;
+ }
+
+ try (var stream = Files.walk(directory, 1)) {
+ for (Path path : stream.filter(Files::isRegularFile).toList()) {
+ if (path.getFileName().toString().endsWith(".kamelet.yaml")) {
+ addTemplateIfSupported(Files.readString(path), templates, seenNames);
+ }
+ }
+ }
+ }
+
+ private void collectFromJar(URL jarDirUrl,
+ List<KameletTemplate> templates,
+ Set<String> seenNames) throws Exception {
+ JarURLConnection connection = (JarURLConnection) jarDirUrl.openConnection();
+ String prefix = connection.getEntryName() == null ? "kamelets/" : ensureTrailingSlash(connection.getEntryName());
+
+ try (JarFile jarFile = connection.getJarFile()) {
+ var entries = jarFile.entries();
+ while (entries.hasMoreElements()) {
+ JarEntry entry = entries.nextElement();
+ String name = entry.getName();
+ if (entry.isDirectory() || !name.startsWith(prefix) || !name.endsWith(".kamelet.yaml")) {
+ continue;
+ }
+ if (name.substring(prefix.length()).contains("/")) {
+ continue;
+ }
+
+ String content = new String(jarFile.getInputStream(entry).readAllBytes(), StandardCharsets.UTF_8);
+ addTemplateIfSupported(content, templates, seenNames);
+ }
+ }
+ }
+
+ private void addTemplateIfSupported(String yamlContent,
+ List<KameletTemplate> templates,
+ Set<String> seenNames) {
+ try {
+ Object parsed = YAML_LOADER.loadFromString(yamlContent);
+ if (!(parsed instanceof Map<?, ?> root)) {
+ return;
+ }
+
+ KameletTemplate template = parseTemplate(root, yamlContent);
+ if (template != null && seenNames.add(template.name())) {
+ templates.add(template);
+ }
+ } catch (Exception e) {
+ LOG.debug("Skipping unreadable Kamelet YAML during template discovery", e);
+ }
+ }
+
+ private KameletTemplate parseTemplate(Map<?, ?> root,
+ String yamlContent) {
+ Map<String, Object> metadata = asStringObjectMap(root.get("metadata"));
+ Map<String, Object> annotations = asStringObjectMap(metadata.get("annotations"));
+ Map<String, Object> spec = asStringObjectMap(root.get("spec"));
+ Map<String, Object> definition = asStringObjectMap(spec.get("definition"));
+ Map<String, Object> template = asStringObjectMap(spec.get("template"));
+ Map<String, Object> from = asStringObjectMap(template.get("from"));
+
+
+ String name = trimmedString(metadata.get("name"));
+ if (name == null || name.isEmpty() || !isSinkKamelet(name, from)) {
+ return null;
+ }
+
+ String displayName = firstNonBlank(trimmedString(definition.get("title")), name);
+ String description = firstNonBlank(trimmedString(definition.get("description")), "");
+ String group = firstNonBlank(trimmedString(annotations.get("camel.apache.org/kamelet.group")), "");
+ String toUriTemplate = getUriTemplateString(from);
+ List<KameletPropertyDefinition> properties = parseProperties(definition);
+
+ return new KameletTemplate(
+ displayName,
+ description,
+ group,
+ name,
+ "kamelet:" + name + "/sink",
+ toUriTemplate,
+ properties,
+ sanitizeYamlSpec(yamlContent).strip(),
+ decodeEmbeddedIcon(annotations)
+ );
+ }
+
+ private String getUriTemplateString(Map<String, Object> from) {
+ return findToUri(asObjectList(from.get("steps"))).orElse("");
+ }
+
+ private java.util.Optional<String> findToUri(List<Object> steps) {
+ for (Object rawStep : steps) {
+ Map<String, Object> step = asStringObjectMap(rawStep);
+ if (step.isEmpty()) {
+ continue;
+ }
+
+ String directToUri = extractToUri(step);
+ if (directToUri != null) {
+ return java.util.Optional.of(directToUri);
+ }
+
+ java.util.Optional<String> nestedToUri = findToUriInNestedBlocks(step);
+ if (nestedToUri.isPresent()) {
+ return nestedToUri;
+ }
+ }
+
+ return java.util.Optional.empty();
+ }
+
+ private String extractToUri(Map<String, Object> step) {
+ Map<String, Object> to = asStringObjectMap(step.get("to"));
+ String toUri = trimmedString(to.get("uri"));
+ if (toUri != null && !toUri.isBlank()) {
+ return toUri;
+ }
+
+ return null;
+ }
+
+ private java.util.Optional<String> findToUriInNestedBlocks(Map<String, Object> step) {
+ for (Object value : step.values()) {
+ java.util.Optional<String> nestedToUri = findToUriInValue(value);
+ if (nestedToUri.isPresent()) {
+ return nestedToUri;
+ }
+ }
+
+ return java.util.Optional.empty();
+ }
+
+ private java.util.Optional<String> findToUriInValue(Object value) {
+ if (value instanceof List<?> list) {
+ return findToUri(new ArrayList<>(list));
+ }
+
+ if (value instanceof Map<?, ?> rawMap) {
+ Map<String, Object> map = asStringObjectMap(rawMap);
+ java.util.Optional<String> fromSteps = findToUri(asObjectList(map.get("steps")));
+ if (fromSteps.isPresent()) {
+ return fromSteps;
+ }
+
+ for (Object nestedValue : map.values()) {
+ java.util.Optional<String> nested = findToUriInValue(nestedValue);
+ if (nested.isPresent()) {
+ return nested;
+ }
+ }
+ }
+
+ return java.util.Optional.empty();
+ }
+
+
+ private String sanitizeYamlSpec(String yamlContent) {
+ String[] lines = removeLeadingCommentBlock(yamlContent)
+ .replace("\r\n", "\n")
+ .split("\n", -1);
+ StringBuilder sanitized = new StringBuilder();
+ boolean skipping = false;
+ int skipIndent = -1;
+
+ for (String line : lines) {
+ if (!skipping && isKameletIconLine(line)) {
+ skipping = true;
+ skipIndent = indentation(line);
+ continue;
+ }
+
+ if (skipping) {
+ if (line.isBlank()) {
+ continue;
+ }
+
+ int currentIndent = indentation(line);
+ if (currentIndent > skipIndent) {
+ continue;
+ }
+
+ skipping = false;
+ }
+
+ sanitized.append(line).append("\n");
+ }
+
+ return sanitized.toString();
+ }
+
+ private String removeLeadingCommentBlock(String yamlContent) {
+ String normalized = yamlContent.replace("\r\n", "\n");
+ String[] lines = normalized.split("\n", -1);
+ int index = 0;
+
+ while (index < lines.length && (lines[index].isBlank() || lines[index].stripLeading().startsWith("#"))) {
+ index++;
+ }
+
+ return String.join("\n", java.util.Arrays.copyOfRange(lines, index, lines.length));
+ }
+
+ private boolean isKameletIconLine(String line) {
+ String trimmed = line.stripLeading();
+ return trimmed.startsWith("camel.apache.org/kamelet.icon:");
+ }
+
+ private int indentation(String line) {
+ int indent = 0;
+ while (indent < line.length() && Character.isWhitespace(line.charAt(indent))) {
+ indent++;
+ }
+ return indent;
+ }
+
+ private byte[] decodeEmbeddedIcon(Map<String, Object> annotations) {
+ String base64Icon = trimmedString(annotations.get("camel.apache.org/kamelet.icon"));
+ return iconDecoder.decode(base64Icon);
+ }
+
+ private List<KameletPropertyDefinition> parseProperties(Map<String, Object> definition) {
+ Map<String, Object> properties = asStringObjectMap(definition.get("properties"));
+ Set<String> required = extractRequired(definition.get("required"));
+ List<KameletPropertyDefinition> result = new ArrayList<>();
+
+ properties.forEach((name, rawSpec) -> {
+ Map<String, Object> spec = asStringObjectMap(rawSpec);
+ List<String> allowedValues = determineAllowedValues(spec);
+ result.add(new KameletPropertyDefinition(
+ name,
+ firstNonBlank(trimmedString(spec.get("title")), name),
+ firstNonBlank(trimmedString(spec.get("description")), ""),
+ determineInputType(name, spec, allowedValues),
+ determineDatatype(spec),
+ required.contains(name),
+ allowedValues,
+ defaultValue(spec.get("default"))
+ ));
+ });
+
+ return result;
+ }
+
+ private KameletPropertyDefinition.PropertyInputType determineInputType(String propertyName,
+ Map<String, Object> spec,
+ List<String> allowedValues) {
+ if (!allowedValues.isEmpty()) {
+ return KameletPropertyDefinition.PropertyInputType.ONE_OF;
+ }
+ if (isSecretProperty(propertyName, spec)) {
+ return KameletPropertyDefinition.PropertyInputType.SECRET;
+ }
+ return KameletPropertyDefinition.PropertyInputType.TEXT;
+ }
+
+ private Datatypes determineDatatype(Map<String, Object> spec) {
+ String type = trimmedString(spec.get("type"));
+ if ("boolean".equals(type)) {
+ return Datatypes.Boolean;
+ } else if ("integer".equals(type)) {
+ return Datatypes.Integer;
+ } else if ("number".equals(type)) {
+ return Datatypes.Double;
+ } else {
+ return Datatypes.String;
+ }
+ }
+
+ private boolean isSecretProperty(String propertyName, Map<String, Object> spec) {
+ String format = trimmedString(spec.get("format"));
+ if ("password".equalsIgnoreCase(format)) {
+ return true;
+ }
+
+ Object descriptors = spec.get("x-descriptors");
+ if (descriptors instanceof List<?> list) {
+ boolean fromDescriptors = list.stream()
+ .map(String::valueOf)
+ .map(String::toLowerCase)
+ .anyMatch(value -> value.contains("password") || value.contains("secret")
+ || value.contains("credential") || value.contains("token"));
+ if (fromDescriptors) {
+ return true;
+ }
+ }
+
+ String loweredName = propertyName.toLowerCase();
+ return loweredName.contains("password")
+ || loweredName.contains("secret")
+ || loweredName.contains("token")
+ || loweredName.contains("apikey")
+ || loweredName.contains("api-key");
+ }
+
+ private boolean isSinkKamelet(String name, Map<String, Object> from) {
+ String fromUri = trimmedString(from.get("uri"));
+ return "kamelet:source".equals(fromUri) || name.endsWith("-sink");
+ }
+
+ private List<String> determineAllowedValues(Map<String, Object> spec) {
+ List<String> enumValues = extractAllowedValues(spec.get("enum"));
+ if (!enumValues.isEmpty()) {
+ return enumValues;
+ }
+
+ String type = trimmedString(spec.get("type"));
+ if ("boolean".equals(type)) {
+ return List.of(Boolean.TRUE.toString(), Boolean.FALSE.toString());
+ }
+
+ return List.of();
+ }
+
+ private List<String> extractAllowedValues(Object value) {
+ if (value instanceof List<?> list) {
+ return list.stream().map(String::valueOf).toList();
+ }
+ return List.of();
+ }
+
+ private Set<String> extractRequired(Object value) {
+ if (value instanceof List<?> list) {
+ return list.stream().map(String::valueOf).collect(Collectors.toSet());
+ }
+ return Set.of();
+ }
+
+ private String defaultValue(Object value) {
+ return value == null ? null : String.valueOf(value);
+ }
+
+ private Map<String, Object> asStringObjectMap(Object value) {
+ if (value instanceof Map<?, ?> raw) {
+ LinkedHashMap<String, Object> result = new LinkedHashMap<>();
+ raw.forEach((k, v) -> result.put(String.valueOf(k), v));
+ return result;
+ }
+ return Map.of();
+ }
+
+ private List<Object> asObjectList(Object value) {
+ if (value instanceof List<?> list) {
+ return new ArrayList<>(list);
+ }
+ return List.of();
+ }
+
+ private String trimmedString(Object value) {
+ return value == null ? null : String.valueOf(value).trim();
+ }
+
+ private String firstNonBlank(String... values) {
+ for (String value : values) {
+ if (value != null && !value.isBlank()) {
+ return value;
+ }
+ }
+ return "";
+ }
+
+ private String ensureTrailingSlash(String prefix) {
+ return prefix.endsWith("/") ? prefix : prefix + "/";
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/factory/KameletSinkFactory.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/factory/KameletSinkFactory.java
new file mode 100644
index 0000000000..603d6d4e4d
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/factory/KameletSinkFactory.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.factory;
+
+import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.assets.KameletSinkAssetResolver;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.provider.KameletTemplateProvider;
+import org.apache.streampipes.extensions.connectors.camel.sink.GeneratedApacheCamelKameletSink;
+
+import java.util.List;
+
+public class KameletSinkFactory {
+
+ private final KameletTemplateProvider templateProvider;
+ private final String appIdPrefix;
+ private final String assetRoot;
+
+ public KameletSinkFactory(KameletTemplateProvider templateProvider,
+ String appIdPrefix,
+ String assetRoot) {
+ this.templateProvider = templateProvider;
+ this.appIdPrefix = appIdPrefix;
+ this.assetRoot = assetRoot;
+ }
+
+ public List<IStreamPipesPipelineElement<?>> createSinks() {
+ return templateProvider.getTemplates()
+ .stream()
+ .map(this::createSink)
+ .toList();
+ }
+
+ protected IStreamPipesPipelineElement<?> createSink(KameletTemplate template) {
+ String appId = makeAppId(template.name());
+ return new GeneratedApacheCamelKameletSink(
+ appId,
+ template.name(),
+ templateProvider,
+ new KameletSinkAssetResolver(appId, template.name(), assetRoot, templateProvider)
+ );
+ }
+
+ protected String makeAppId(String templateName) {
+ return appIdPrefix + "." + templateName;
+ }
+
+ protected KameletTemplateProvider getTemplateProvider() {
+ return templateProvider;
+ }
+
+ protected String getAssetRoot() {
+ return assetRoot;
+ }
+}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/filter/KameletTemplateFilter.java
similarity index 62%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/filter/KameletTemplateFilter.java
index e01f7adeb1..50effc195e 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/filter/KameletTemplateFilter.java
@@ -16,17 +16,20 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.extensions.connectors.camel.kamelet.filter;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
-import java.util.function.Supplier;
+@FunctionalInterface
+public interface KameletTemplateFilter {
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
+ boolean test(KameletTemplate template);
- NeT getDescription();
+ default KameletTemplateFilter and(KameletTemplateFilter other) {
+ return template -> test(template) && other.test(template);
+ }
- Supplier<PeT> getSupplier();
+ static KameletTemplateFilter includeAll() {
+ return template -> true;
+ }
}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/CamelMessage.java
similarity index 65%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/CamelMessage.java
index e01f7adeb1..5de5e6e276 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/CamelMessage.java
@@ -16,17 +16,14 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.extensions.connectors.camel.kamelet.message;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import java.util.Map;
-import java.util.function.Supplier;
+public record CamelMessage(Object body,
+ Map<String, Object> headers) {
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
-
- NeT getDescription();
-
- Supplier<PeT> getSupplier();
+ public CamelMessage {
+ headers = headers == null ? Map.of() : Map.copyOf(headers);
+ }
}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/KameletEventMessageMapper.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/KameletEventMessageMapper.java
new file mode 100644
index 0000000000..fee79dde77
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/KameletEventMessageMapper.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.message;
+
+import org.apache.streampipes.dataformat.JsonDataFormatDefinition;
+import org.apache.streampipes.model.runtime.Event;
+import org.apache.streampipes.model.runtime.field.AbstractField;
+
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class KameletEventMessageMapper {
+
+ public static final String EVENT_JSON_HEADER = "streampipes.event.json";
+ public static final String EVENT_BODY_TYPE_HEADER = "streampipes.event.bodyType";
+ public static final String EVENT_BODY_FIELD_SELECTOR_HEADER = "streampipes.event.bodyFieldSelector";
+
+ public CamelMessage mapEvent(Event event,
+ KameletMessageMapping messageMapping,
+ JsonDataFormatDefinition jsonDataFormatDefinition) {
+ Map<String, Object> rawEvent = event.getRaw();
+ String jsonPayload = new String(jsonDataFormatDefinition.fromMap(rawEvent), StandardCharsets.UTF_8);
+
+ Object body = switch (messageMapping.payloadMode()) {
+ case EVENT_MAP -> rawEvent;
+ case EVENT_JSON -> jsonPayload;
+ case MAPPED_FIELD -> extractEventValue(event, messageMapping.bodyFieldSelector());
+ };
+
+ Map<String, Object> headers = new LinkedHashMap<>();
+ headers.put(EVENT_JSON_HEADER, jsonPayload);
+ headers.put(EVENT_BODY_TYPE_HEADER, bodyType(messageMapping.payloadMode()));
+
+ if (messageMapping.payloadMode() == KameletMessageMapping.PayloadMode.MAPPED_FIELD
+ && messageMapping.bodyFieldSelector() != null
+ && !messageMapping.bodyFieldSelector().isBlank()) {
+ headers.put(EVENT_BODY_FIELD_SELECTOR_HEADER, messageMapping.bodyFieldSelector());
+ }
+
+ for (KameletHeaderMapping headerMapping : messageMapping.headerMappings()) {
+ Object headerValue = extractEventValue(event, headerMapping.eventFieldSelector());
+ if (headerValue != null) {
+ headers.put(headerMapping.headerName(), headerValue);
+ }
+ }
+
+ return new CamelMessage(body, headers);
+ }
+
+ private Object extractEventValue(Event event,
+ String selector) {
+ if (selector == null || selector.isBlank()) {
+ return null;
+ }
+
+ return toPlainValue(event.getFieldBySelector(selector));
+ }
+
+ private Object toPlainValue(AbstractField<?> field) {
+ if (field == null) {
+ return null;
+ }
+
+ if (field.isPrimitive()) {
+ return field.getRawValue();
+ }
+
+ if (field.isList()) {
+ return field.getAsList()
+ .getRawValue()
+ .stream()
+ .map(this::toPlainValue)
+ .toList();
+ }
+
+ Map<String, Object> nested = new LinkedHashMap<>();
+ field.getAsComposite()
+ .getRawValue()
+ .values()
+ .forEach(child -> nested.put(child.getFieldNameIn(), toPlainValue(child)));
+ return nested;
+ }
+
+ private String bodyType(KameletMessageMapping.PayloadMode payloadMode) {
+ return switch (payloadMode) {
+ case EVENT_MAP -> "map";
+ case EVENT_JSON -> "json";
+ case MAPPED_FIELD -> "mapped-field";
+ };
+ }
+}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/KameletHeaderMapping.java
similarity index 65%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/KameletHeaderMapping.java
index e01f7adeb1..ee9fdb50b8 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/KameletHeaderMapping.java
@@ -16,17 +16,8 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.extensions.connectors.camel.kamelet.message;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
-
-import java.util.function.Supplier;
-
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
-
- NeT getDescription();
-
- Supplier<PeT> getSupplier();
+public record KameletHeaderMapping(String headerName,
+ String eventFieldSelector) {
}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/KameletMessageMapping.java
similarity index 58%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/KameletMessageMapping.java
index e01f7adeb1..f2501a1f95 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/message/KameletMessageMapping.java
@@ -16,17 +16,22 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.extensions.connectors.camel.kamelet.message;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import java.util.List;
-import java.util.function.Supplier;
+public record KameletMessageMapping(PayloadMode payloadMode,
+ String bodyFieldSelector,
+ List<KameletHeaderMapping> headerMappings,
+ String transformStepsYaml) {
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
+ public KameletMessageMapping {
+ headerMappings = headerMappings == null ? List.of() : List.copyOf(headerMappings);
+ }
- NeT getDescription();
-
- Supplier<PeT> getSupplier();
+ public enum PayloadMode {
+ EVENT_MAP,
+ EVENT_JSON,
+ MAPPED_FIELD
+ }
}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/model/KameletPropertyDefinition.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/model/KameletPropertyDefinition.java
new file mode 100644
index 0000000000..9b0afffd08
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/model/KameletPropertyDefinition.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.model;
+
+import org.apache.streampipes.sdk.utils.Datatypes;
+
+import java.util.List;
+
+public record KameletPropertyDefinition(String name,
+ String label,
+ String description,
+ PropertyInputType inputType,
+ Datatypes datatype,
+ boolean required,
+ List<String> allowedValues,
+ String defaultValue) {
+
+ public String staticPropertyInternalName(String templateName) {
+ return "camel-kamelet-param-" + sanitize(templateName) + "-" + sanitize(name);
+ }
+
+ public String displayLabel() {
+ return label == null || label.isBlank() ? name : label;
+ }
+
+ private String sanitize(String value) {
+ return value == null ? "" : value.replaceAll("[^a-zA-Z0-9]+", "-").replaceAll("(^-|-$)", "").toLowerCase();
+ }
+
+ public enum PropertyInputType {
+ TEXT,
+ SECRET,
+ ONE_OF
+ }
+}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/model/KameletTemplate.java
similarity index 58%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/model/KameletTemplate.java
index e01f7adeb1..5c73558f36 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/model/KameletTemplate.java
@@ -16,17 +16,17 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.extensions.connectors.camel.kamelet.model;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import java.util.List;
-import java.util.function.Supplier;
-
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
-
- NeT getDescription();
-
- Supplier<PeT> getSupplier();
+public record KameletTemplate(String displayName,
+ String description,
+ String group,
+ String name,
+ String endpointBaseUri,
+ String toUriTemplate,
+ List<KameletPropertyDefinition> properties,
+ String yamlSpec,
+ byte[] embeddedIcon) {
}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/module/AbstractCamelKameletModuleExport.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/module/AbstractCamelKameletModuleExport.java
new file mode 100644
index 0000000000..702d0cf91f
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/module/AbstractCamelKameletModuleExport.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.module;
+
+import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter;
+import org.apache.streampipes.extensions.api.declarer.IExtensionModuleExport;
+import org.apache.streampipes.extensions.api.migration.IModelMigrator;
+import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.factory.KameletSinkFactory;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.filter.KameletTemplateFilter;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.provider.ClasspathKameletTemplateProvider;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.provider.KameletTemplateProvider;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.List;
+
+public abstract class AbstractCamelKameletModuleExport implements IExtensionModuleExport {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractCamelKameletModuleExport.class);
+
+ private List<IStreamPipesPipelineElement<?>> pipelineElements;
+
+ @Override
+ public List<StreamPipesAdapter> adapters() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public synchronized List<IStreamPipesPipelineElement<?>> pipelineElements() {
+ if (pipelineElements == null) {
+ KameletTemplateProvider templateProvider = createTemplateProvider();
+ pipelineElements = createSinkFactory(templateProvider).createSinks();
+
+ if (pipelineElements.isEmpty()) {
+ LOG.warn("No Camel Kamelet sinks discovered for module {}", getClass().getSimpleName());
+ }
+ }
+
+ return pipelineElements;
+ }
+
+ @Override
+ public List<IModelMigrator<?, ?>> migrators() {
+ return Collections.emptyList();
+ }
+
+ protected KameletTemplateProvider createTemplateProvider() {
+ return new ClasspathKameletTemplateProvider(getTemplateFilter());
+ }
+
+ protected KameletSinkFactory createSinkFactory(KameletTemplateProvider templateProvider) {
+ return new KameletSinkFactory(templateProvider, getAppIdPrefix(), getAssetRoot());
+ }
+
+ protected String getAssetRoot() {
+ return "camel-kamelets";
+ }
+
+ protected abstract String getAppIdPrefix();
+
+ protected abstract KameletTemplateFilter getTemplateFilter();
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/provider/ClasspathKameletTemplateProvider.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/provider/ClasspathKameletTemplateProvider.java
new file mode 100644
index 0000000000..7802633ccb
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/provider/ClasspathKameletTemplateProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.provider;
+
+import org.apache.streampipes.commons.exceptions.SpConfigurationException;
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.discovery.KameletTemplateDiscovery;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.filter.KameletTemplateFilter;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class ClasspathKameletTemplateProvider implements KameletTemplateProvider {
+
+ private final KameletTemplateDiscovery templateDiscovery;
+ private final KameletTemplateFilter templateFilter;
+
+ private Map<String, KameletTemplate> templates;
+
+ public ClasspathKameletTemplateProvider(KameletTemplateFilter templateFilter) {
+ this.templateDiscovery = new KameletTemplateDiscovery();
+ this.templateFilter = templateFilter;
+ }
+
+ @Override
+ public synchronized List<KameletTemplate> getTemplates() {
+ return List.copyOf(loadTemplates().values());
+ }
+
+ @Override
+ public synchronized Optional<KameletTemplate> getTemplate(String templateName) {
+ return Optional.ofNullable(loadTemplates().get(templateName));
+ }
+
+ private Map<String, KameletTemplate> loadTemplates() {
+ if (templates == null) {
+ try {
+ templates = new LinkedHashMap<>(templateDiscovery.discoverTemplates(templateFilter));
+ } catch (SpConfigurationException e) {
+ throw new SpRuntimeException("Could not discover registered Kamelet templates", e);
+ }
+ }
+
+ return templates;
+ }
+}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/provider/KameletTemplateProvider.java
similarity index 55%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/provider/KameletTemplateProvider.java
index e01f7adeb1..4670d375c7 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/provider/KameletTemplateProvider.java
@@ -16,17 +16,22 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.extensions.connectors.camel.kamelet.provider;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
-import java.util.function.Supplier;
+import java.util.List;
+import java.util.Optional;
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
+public interface KameletTemplateProvider {
- NeT getDescription();
+ List<KameletTemplate> getTemplates();
- Supplier<PeT> getSupplier();
+ Optional<KameletTemplate> getTemplate(String templateName);
+
+ default KameletTemplate requireTemplate(String templateName) {
+ return getTemplate(templateName)
+ .orElseThrow(() -> new SpRuntimeException("Could not find registered Kamelet template: " + templateName));
+ }
}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/transform/KameletTransformRouteLoader.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/transform/KameletTransformRouteLoader.java
new file mode 100644
index 0000000000..3f7ec7ab46
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/kamelet/transform/KameletTransformRouteLoader.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.kamelet.transform;
+
+import org.apache.streampipes.commons.exceptions.SpConfigurationException;
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+
+import org.apache.camel.CamelContext;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.dsl.yaml.YamlRoutesBuilderLoader;
+import org.apache.camel.spi.Resource;
+import org.snakeyaml.engine.v2.api.Load;
+import org.snakeyaml.engine.v2.api.LoadSettings;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class KameletTransformRouteLoader {
+
+ private static final Load YAML_LOADER = new Load(LoadSettings.builder().build());
+ private static final Set<String> FORBIDDEN_KEYS = Set.of(
+ "route",
+ "from",
+ "templatedRoute",
+ "routeTemplate",
+ "routeConfiguration",
+ "routeConfigurations",
+ "rest",
+ "beans"
+ );
+
+ public void addRoute(CamelContext camelContext,
+ String routeId,
+ String fromUri,
+ String toUri,
+ String transformStepsYaml) throws Exception {
+ if (transformStepsYaml == null || transformStepsYaml.isBlank()) {
+ camelContext.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from(fromUri)
+ .routeId(routeId)
+ .to(toUri);
+ }
+ });
+ return;
+ }
+
+ validateSteps(transformStepsYaml);
+
+ YamlRoutesBuilderLoader loader = new YamlRoutesBuilderLoader();
+ loader.setCamelContext(camelContext);
+
+ camelContext.addRoutes(loader.doLoadRouteBuilder(new InMemoryResource(
+ "in-memory:" + routeId + ".yaml",
+ buildRouteYaml(routeId, fromUri, toUri, transformStepsYaml)
+ )));
+ }
+
+ private void validateSteps(String transformStepsYaml) {
+ Object parsed = YAML_LOADER.loadFromString(transformStepsYaml);
+ if (!(parsed instanceof List<?> list)) {
+ throw new SpRuntimeException(new SpConfigurationException(
+ "Advanced transform steps must be a YAML list of Camel steps."
+ ));
+ }
+
+ validateNode(list);
+ }
+
+ private void validateNode(Object node) {
+ if (node instanceof List<?> list) {
+ list.forEach(this::validateNode);
+ } else if (node instanceof Map<?, ?> map) {
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ String key = String.valueOf(entry.getKey());
+ if (FORBIDDEN_KEYS.contains(key)) {
+ throw new SpRuntimeException(new SpConfigurationException(
+ "Advanced transform steps may not declare Camel routes, beans or from clauses."
+ ));
+ }
+ validateNode(entry.getValue());
+ }
+ }
+ }
+
+ private String buildRouteYaml(String routeId,
+ String fromUri,
+ String toUri,
+ String transformStepsYaml) {
+ return """
+ - route:
+ id: '%s'
+ from:
+ uri: '%s'
+ steps:
+ %s
+ - to:
+ uri: '%s'
+ """.formatted(
+ escapeYaml(routeId),
+ escapeYaml(fromUri),
+ indentSteps(transformStepsYaml),
+ escapeYaml(toUri)
+ );
+ }
+
+ private String indentSteps(String transformStepsYaml) {
+ StringBuilder builder = new StringBuilder();
+ String[] lines = transformStepsYaml.replace("\r\n", "\n").split("\n", -1);
+
+ for (String line : lines) {
+ if (line.isBlank()) {
+ builder.append('\n');
+ } else {
+ builder.append(" ").append(line).append('\n');
+ }
+ }
+
+ return builder.toString();
+ }
+
+ private String escapeYaml(String value) {
+ return value.replace("'", "''");
+ }
+
+ private static final class InMemoryResource implements Resource {
+
+ private final String location;
+ private final byte[] content;
+
+ private InMemoryResource(String location,
+ String content) {
+ this.location = location;
+ this.content = content.getBytes(StandardCharsets.UTF_8);
+ }
+
+ @Override
+ public String getScheme() {
+ return "in-memory";
+ }
+
+ @Override
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public boolean exists() {
+ return true;
+ }
+
+ @Override
+ public InputStream getInputStream() {
+ return new ByteArrayInputStream(content);
+ }
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/sink/ApacheCamelKameletSink.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/sink/ApacheCamelKameletSink.java
new file mode 100644
index 0000000000..b46958aa8d
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/sink/ApacheCamelKameletSink.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.sink;
+
+import org.apache.streampipes.commons.exceptions.SpConfigurationException;
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.dataformat.JsonDataFormatDefinition;
+import org.apache.streampipes.extensions.api.assets.AssetResolver;
+import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor;
+import org.apache.streampipes.extensions.api.pe.IStreamPipesDataSink;
+import org.apache.streampipes.extensions.api.pe.config.IDataSinkConfiguration;
+import org.apache.streampipes.extensions.api.pe.context.EventSinkRuntimeContext;
+import org.apache.streampipes.extensions.api.pe.param.IDataSinkParameters;
+import org.apache.streampipes.extensions.api.runtime.SupportsRuntimeConfig;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.config.KameletConfigurationExtractor;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.config.KameletMessageMappingExtractor;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.config.KameletSinkStaticPropertyProvider;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.config.KameletStaticPropertyGenerator;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.message.CamelMessage;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.message.KameletEventMessageMapper;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.message.KameletMessageMapping;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.provider.KameletTemplateProvider;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.transform.KameletTransformRouteLoader;
+import org.apache.streampipes.model.DataSinkType;
+import org.apache.streampipes.model.extensions.ExtensionAssetType;
+import org.apache.streampipes.model.runtime.Event;
+import org.apache.streampipes.model.staticproperty.RuntimeResolvableGroupStaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+import org.apache.streampipes.sdk.builder.DataSinkBuilder;
+import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.apache.streampipes.sdk.builder.sink.DataSinkConfiguration;
+import org.apache.streampipes.sdk.helpers.EpRequirements;
+import org.apache.streampipes.sdk.helpers.Locales;
+
+import org.apache.camel.CamelContext;
+import org.apache.camel.Exchange;
+import org.apache.camel.ProducerTemplate;
+import org.apache.camel.impl.DefaultCamelContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+public abstract class ApacheCamelKameletSink implements IStreamPipesDataSink, SupportsRuntimeConfig {
+
+ public static final String KAMELET_PARAMETERS_GROUP_KEY = KameletSinkStaticPropertyProvider.KAMELET_PARAMETERS_GROUP_KEY;
+
+ private static final Logger LOG = LoggerFactory.getLogger(ApacheCamelKameletSink.class);
+ private static final String ENTRY_ENDPOINT_URI = "direct:streampipes-kamelet-input";
+ private static final String ROUTE_ID = "streampipes-kamelet-sink-route";
+
+ private final String appId;
+ private final String templateName;
+ private final AssetResolver assetResolver;
+ private final JsonDataFormatDefinition jsonDataFormatDefinition;
+ private final KameletTemplateProvider templateProvider;
+ private final KameletSinkStaticPropertyProvider staticPropertyProvider;
+ private final KameletStaticPropertyGenerator staticPropertyGenerator;
+ private final KameletConfigurationExtractor configurationExtractor;
+ private final KameletMessageMappingExtractor messageMappingExtractor;
+ private final KameletEventMessageMapper eventMessageMapper;
+ private final KameletTransformRouteLoader transformRouteLoader;
+
+ private CamelContext camelContext;
+ private ProducerTemplate producerTemplate;
+ private String kameletEndpointUri;
+ private KameletMessageMapping messageMapping;
+
+ protected ApacheCamelKameletSink(String appId,
+ String templateName,
+ KameletTemplateProvider templateProvider,
+ AssetResolver assetResolver) {
+ this.appId = appId;
+ this.templateName = templateName;
+ this.templateProvider = templateProvider;
+ this.assetResolver = assetResolver;
+ this.jsonDataFormatDefinition = new JsonDataFormatDefinition();
+ this.staticPropertyProvider = new KameletSinkStaticPropertyProvider();
+ this.staticPropertyGenerator = new KameletStaticPropertyGenerator();
+ this.configurationExtractor = new KameletConfigurationExtractor();
+ this.messageMappingExtractor = new KameletMessageMappingExtractor();
+ this.eventMessageMapper = new KameletEventMessageMapper();
+ this.transformRouteLoader = new KameletTransformRouteLoader();
+ }
+
+ @Override
+ public IDataSinkConfiguration declareConfig() {
+ return DataSinkConfiguration.create(
+ this::newInstance,
+ DataSinkBuilder.create(appId, 0)
+ .category(DataSinkType.FORWARD)
+ .withLocales(Locales.EN)
+ .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON)
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredProperty(EpRequirements.anyProperty())
+ .build())
+ .requiredStaticProperty(staticPropertyProvider.createKameletParameterGroup())
+ .requiredStaticProperty(staticPropertyProvider.createMessageMappingGroup())
+ .requiredStaticProperty(staticPropertyProvider.createAdvancedTransformAlternatives())
+ .build(),
+ assetResolver
+ );
+ }
+
+ @Override
+ public void onPipelineStarted(IDataSinkParameters parameters,
+ EventSinkRuntimeContext runtimeContext) {
+ KameletTemplate template = resolveTemplate();
+ RuntimeResolvableGroupStaticProperty parameterGroup = parameters.extractor().getStaticPropertyByName(
+ KAMELET_PARAMETERS_GROUP_KEY,
+ RuntimeResolvableGroupStaticProperty.class
+ );
+
+ this.messageMapping = messageMappingExtractor.extract(parameters.extractor());
+ var params = configurationExtractor.buildParams(template, parameterGroup);
+ this.kameletEndpointUri = template.endpointBaseUri();
+
+ try {
+ this.camelContext = createCamelContext(template, params);
+ this.camelContext.start();
+
+ this.producerTemplate = this.camelContext.createProducerTemplate();
+ this.producerTemplate.start();
+
+ LOG.info("Started Apache Camel Kamelet sink {} -> {}", template.name(), this.kameletEndpointUri);
+ } catch (Exception e) {
+ stopQuietly();
+ throw new SpRuntimeException("Could not start Apache Camel Kamelet sink", e);
+ }
+ }
+
+ @Override
+ public void onEvent(Event event) {
+ if (producerTemplate == null) {
+ LOG.warn("Apache Camel Kamelet sink is not initialized yet. Dropping event.");
+ return;
+ }
+
+ try {
+ CamelMessage camelMessage = eventMessageMapper.mapEvent(event, messageMapping, jsonDataFormatDefinition);
+ Exchange exchange = producerTemplate.send(ENTRY_ENDPOINT_URI, exchangeBuilder -> {
+ exchangeBuilder.getIn().setBody(camelMessage.body());
+ for (Map.Entry<String, Object> header : camelMessage.headers().entrySet()) {
+ exchangeBuilder.getIn().setHeader(header.getKey(), header.getValue());
+ }
+ });
+
+ if (exchange.getException() != null) {
+ throw exchange.getException();
+ }
+ } catch (Exception e) {
+ LOG.error("Could not route event through Camel Kamelet endpoint {}", kameletEndpointUri, e);
+ throw new SpRuntimeException("Could not route event through Camel Kamelet endpoint", e);
+ }
+ }
+
+ @Override
+ public void onPipelineStopped() {
+ stopQuietly();
+ }
+
+ @Override
+ public StaticProperty resolveConfiguration(String staticPropertyInternalName,
+ IStaticPropertyExtractor extractor) throws SpConfigurationException {
+ if (KAMELET_PARAMETERS_GROUP_KEY.equals(staticPropertyInternalName)) {
+ RuntimeResolvableGroupStaticProperty group = extractor.getStaticPropertyByName(
+ KAMELET_PARAMETERS_GROUP_KEY,
+ RuntimeResolvableGroupStaticProperty.class
+ );
+ return staticPropertyGenerator.configureParameterGroup(group, resolveTemplate());
+ }
+
+ throw new SpConfigurationException("Unsupported runtime-resolvable property: " + staticPropertyInternalName);
+ }
+
+ protected abstract ApacheCamelKameletSink newInstance();
+
+ protected String getAppId() {
+ return appId;
+ }
+
+ protected String getTemplateName() {
+ return templateName;
+ }
+
+ protected KameletTemplateProvider getTemplateProvider() {
+ return templateProvider;
+ }
+
+ protected AssetResolver getAssetResolver() {
+ return assetResolver;
+ }
+
+ protected void customizeCamelContext(CamelContext context,
+ KameletTemplate template,
+ Map<String, Object> params) throws Exception {
+ }
+
+ protected void onSinkStopped() throws Exception {
+ }
+
+ private KameletTemplate resolveTemplate() {
+ return templateProvider.requireTemplate(templateName);
+ }
+
+ private CamelContext createCamelContext(KameletTemplate template,
+ Map<String, Object> params) throws Exception {
+ String routeId = ROUTE_ID + "-" + templateName;
+ CamelContext context = new DefaultCamelContext();
+ params.forEach((key, value) -> {
+ String prefixKey = "camel.kamelet." + templateName + "." + key;
+ context.getPropertiesComponent().addInitialProperty(prefixKey, String.valueOf(value));
+ });
+ customizeCamelContext(context, template, params);
+ transformRouteLoader.addRoute(
+ context,
+ routeId,
+ ENTRY_ENDPOINT_URI,
+ kameletEndpointUri,
+ messageMapping == null ? null : messageMapping.transformStepsYaml()
+ );
+ return context;
+ }
+
+ private void stopQuietly() {
+ if (producerTemplate != null) {
+ try {
+ producerTemplate.stop();
+ } catch (Exception e) {
+ LOG.warn("Error while stopping Camel producer template", e);
+ } finally {
+ producerTemplate = null;
+ }
+ }
+
+ if (camelContext != null) {
+ try {
+ camelContext.stop();
+ } catch (Exception e) {
+ LOG.warn("Error while stopping Camel context", e);
+ } finally {
+ camelContext = null;
+ }
+ }
+
+ try {
+ onSinkStopped();
+ } catch (Exception e) {
+ LOG.warn("Error while cleaning up Camel Kamelet sink resources", e);
+ }
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/sink/GeneratedApacheCamelKameletSink.java b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/sink/GeneratedApacheCamelKameletSink.java
new file mode 100644
index 0000000000..10cef476d7
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-camel/src/main/java/org/apache/streampipes/extensions/connectors/camel/sink/GeneratedApacheCamelKameletSink.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.camel.sink;
+
+import org.apache.streampipes.extensions.api.assets.AssetResolver;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.provider.KameletTemplateProvider;
+
+public class GeneratedApacheCamelKameletSink extends ApacheCamelKameletSink {
+
+ public GeneratedApacheCamelKameletSink(String appId,
+ String templateName,
+ KameletTemplateProvider templateProvider,
+ AssetResolver assetResolver) {
+ super(appId, templateName, templateProvider, assetResolver);
+ }
+
+ @Override
+ protected ApacheCamelKameletSink newInstance() {
+ return new GeneratedApacheCamelKameletSink(
+ getAppId(),
+ getTemplateName(),
+ getTemplateProvider(),
+ getAssetResolver()
+ );
+ }
+}
diff --git a/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml b/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml
index d1efcf6b06..a9c7ad68c5 100644
--- a/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml
+++ b/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml
@@ -30,6 +30,21 @@
<maven.deploy.skip>true</maven.deploy.skip>
</properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>4.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ <version>5.3.1</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>org.apache.streampipes</groupId>
@@ -54,6 +69,11 @@
<artifactId>streampipes-connect-adapters-iiot</artifactId>
<version>0.99.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-connectors-camel-azure</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.streampipes</groupId>
<artifactId>streampipes-connectors-influx</artifactId>
diff --git a/streampipes-extensions/streampipes-extensions-all-jvm/src/main/java/org/apache/streampipes/extensions/all/jvm/AllExtensionsInit.java b/streampipes-extensions/streampipes-extensions-all-jvm/src/main/java/org/apache/streampipes/extensions/all/jvm/AllExtensionsInit.java
index 4f23d6b9e9..8d82a69d14 100644
--- a/streampipes-extensions/streampipes-extensions-all-jvm/src/main/java/org/apache/streampipes/extensions/all/jvm/AllExtensionsInit.java
+++ b/streampipes-extensions/streampipes-extensions-all-jvm/src/main/java/org/apache/streampipes/extensions/all/jvm/AllExtensionsInit.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.extensions.all.jvm;
import org.apache.streampipes.connect.GeneralAdaptersExtensionModuleExport;
import org.apache.streampipes.connect.iiot.IIoTAdaptersExtensionModuleExport;
import org.apache.streampipes.connectors.ros.RosConnectorsModuleExport;
+import org.apache.streampipes.extensions.connectors.camel.azure.CamelAzureConnectorsModuleExport;
import org.apache.streampipes.extensions.connectors.influx.InfluxConnectorsModuleExport;
import org.apache.streampipes.extensions.connectors.kafka.KafkaConnectorsModuleExport;
import org.apache.streampipes.extensions.connectors.mqtt.MqttConnectorsModuleExport;
@@ -68,6 +69,7 @@ public class AllExtensionsInit extends StreamPipesExtensionsServiceBase {
new GeneralAdaptersExtensionModuleExport(),
new IIoTAdaptersExtensionModuleExport(),
+ new CamelAzureConnectorsModuleExport(),
new InfluxConnectorsModuleExport(),
new KafkaConnectorsModuleExport(),
new MqttConnectorsModuleExport(),
diff --git a/streampipes-integration-tests/pom.xml b/streampipes-integration-tests/pom.xml
index f2f23f10c7..43aa893ba8 100644
--- a/streampipes-integration-tests/pom.xml
+++ b/streampipes-integration-tests/pom.xml
@@ -105,6 +105,18 @@
<version>0.99.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-connectors-camel-azure</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/containers/AzureCosmosDbEmulatorContainer.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/containers/AzureCosmosDbEmulatorContainer.java
new file mode 100644
index 0000000000..99150f54c2
--- /dev/null
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/containers/AzureCosmosDbEmulatorContainer.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.integration.containers;
+
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+
+public class AzureCosmosDbEmulatorContainer extends GenericContainer<AzureCosmosDbEmulatorContainer> {
+
+ private static final DockerImageName IMAGE_NAME =
+ DockerImageName.parse("mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview");
+ public static final String DEFAULT_EMULATOR_KEY =
+ "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==";
+ public static final int DEFAULT_HOST_HTTPS_PORT = 8081;
+ public static final int DEFAULT_HOST_EXPLORER_PORT = 1234;
+
+ private static final int HTTPS_PORT = 8081;
+ private static final int EXPLORER_PORT = 1234;
+ private static final String HTTPS_PORT_PROPERTY = "streampipes.it.cosmosdb.https.port";
+ private static final String EXPLORER_PORT_PROPERTY = "streampipes.it.cosmosdb.explorer.port";
+
+ private final int hostHttpsPort;
+ private final int hostExplorerPort;
+
+ public AzureCosmosDbEmulatorContainer() {
+ super(IMAGE_NAME);
+
+ this.hostHttpsPort = resolvePort(HTTPS_PORT_PROPERTY, "STREAMPIPES_IT_COSMOSDB_HTTPS_PORT", DEFAULT_HOST_HTTPS_PORT);
+ this.hostExplorerPort = resolvePort(
+ EXPLORER_PORT_PROPERTY,
+ "STREAMPIPES_IT_COSMOSDB_EXPLORER_PORT",
+ DEFAULT_HOST_EXPLORER_PORT
+ );
+
+ this.withEnv("PROTOCOL", "https");
+ this.withEnv("ENABLE_EXPLORER", "true");
+ this.addFixedExposedPort(hostHttpsPort, HTTPS_PORT);
+ this.addFixedExposedPort(hostExplorerPort, EXPLORER_PORT);
+ this.waitingFor(Wait.forListeningPort());
+ this.withStartupTimeout(Duration.ofMinutes(5));
+ }
+
+ public String getEmulatorEndpoint() {
+ return "https://" + getHost() + ":" + hostHttpsPort + "/";
+ }
+
+ public String getCertificateEndpoint() {
+ return getEmulatorEndpoint() + "_explorer/emulator.pem";
+ }
+
+ public String getExplorerEndpoint() {
+ return "https://" + getHost() + ":" + hostExplorerPort + "/_explorer/index.html";
+ }
+
+ public String getConfiguredEmulatorKey() {
+ String propertyValue = System.getProperty("streampipes.it.cosmosdb.key");
+ if (propertyValue != null && !propertyValue.isBlank()) {
+ return propertyValue;
+ }
+
+ String envValue = System.getenv("STREAMPIPES_IT_COSMOSDB_KEY");
+ if (envValue != null && !envValue.isBlank()) {
+ return envValue;
+ }
+
+ return DEFAULT_EMULATOR_KEY;
+ }
+
+ private int resolvePort(String propertyName, String envName, int defaultPort) {
+ String propertyValue = System.getProperty(propertyName);
+ if (propertyValue != null && !propertyValue.isBlank()) {
+ return Integer.parseInt(propertyValue);
+ }
+
+ String envValue = System.getenv(envName);
+ if (envValue != null && !envValue.isBlank()) {
+ return Integer.parseInt(envValue);
+ }
+
+ return defaultPort;
+ }
+}
diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/DataSinkTestConfigurator.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/DataSinkTestConfigurator.java
new file mode 100644
index 0000000000..6d364e2fb0
--- /dev/null
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/DataSinkTestConfigurator.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.integration.sinks;
+
+import org.apache.streampipes.commons.exceptions.SpConfigurationException;
+import org.apache.streampipes.extensions.api.pe.IStreamPipesDataSink;
+import org.apache.streampipes.extensions.api.runtime.SupportsRuntimeConfig;
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.staticproperty.CollectionStaticProperty;
+import org.apache.streampipes.model.staticproperty.RuntimeResolvableAnyStaticProperty;
+import org.apache.streampipes.model.staticproperty.RuntimeResolvableGroupStaticProperty;
+import org.apache.streampipes.model.staticproperty.RuntimeResolvableOneOfStaticProperty;
+import org.apache.streampipes.model.staticproperty.RuntimeResolvableTreeInputStaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticPropertyAlternative;
+import org.apache.streampipes.model.staticproperty.StaticPropertyAlternatives;
+import org.apache.streampipes.model.staticproperty.StaticPropertyGroup;
+import org.apache.streampipes.model.util.Cloner;
+import org.apache.streampipes.sdk.extractor.StaticPropertyExtractor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Predicate;
+
+public class DataSinkTestConfigurator {
+
+ public List<StaticProperty> prepareStaticProperties(IStreamPipesDataSink sink,
+ List<SpDataStream> inputStreams)
+ throws SpConfigurationException {
+ List<StaticProperty> staticProperties = new Cloner().staticProperties(sink.declareConfig()
+ .getDescription()
+ .getStaticProperties());
+
+ if (!(sink instanceof SupportsRuntimeConfig supportsRuntimeConfig)) {
+ return staticProperties;
+ }
+
+ List<StaticProperty> resolvedProperties = new ArrayList<>(staticProperties);
+ for (int i = 0; i < resolvedProperties.size(); i++) {
+ StaticProperty staticProperty = resolvedProperties.get(i);
+ if (isRuntimeResolvable(staticProperty)) {
+ resolvedProperties.set(
+ i,
+ supportsRuntimeConfig.resolveConfiguration(
+ staticProperty.getInternalName(),
+ StaticPropertyExtractor.from(
+ resolvedProperties,
+ inputStreams,
+ sink.declareConfig().getDescription().getAppId()
+ )
+ )
+ );
+ }
+ }
+
+ return resolvedProperties;
+ }
+
+ public <T extends StaticProperty> T requireProperty(List<StaticProperty> staticProperties,
+ String internalName,
+ Class<T> propertyType) {
+ return requireProperty(staticProperties, property -> internalName.equals(property.getInternalName()), propertyType);
+ }
+
+ public <T extends StaticProperty> T requireProperty(StaticPropertyGroup group,
+ String internalName,
+ Class<T> propertyType) {
+ return requireProperty(group.getStaticProperties(), internalName, propertyType);
+ }
+
+ public <T extends StaticProperty> T requirePropertyMatching(StaticPropertyGroup group,
+ Predicate<StaticProperty> predicate,
+ Class<T> propertyType) {
+ return requireProperty(group.getStaticProperties(), predicate, propertyType);
+ }
+
+ public void selectAlternative(StaticPropertyAlternatives alternatives,
+ String internalName) {
+ boolean found = false;
+ for (StaticPropertyAlternative alternative : alternatives.getAlternatives()) {
+ boolean selected = internalName.equals(alternative.getInternalName());
+ alternative.setSelected(selected);
+ if (selected) {
+ found = true;
+ }
+ }
+
+ if (!found) {
+ throw new IllegalArgumentException("Could not find alternative " + internalName);
+ }
+ }
+
+ public StaticPropertyGroup addCollectionMember(CollectionStaticProperty collectionStaticProperty) {
+ StaticPropertyGroup member =
+ (StaticPropertyGroup) new Cloner().staticProperty(collectionStaticProperty.getStaticPropertyTemplate());
+ if (collectionStaticProperty.getMembers() == null) {
+ collectionStaticProperty.setMembers(new ArrayList<>());
+ }
+ collectionStaticProperty.getMembers().add(member);
+ return member;
+ }
+
+ private <T extends StaticProperty> T requireProperty(List<StaticProperty> staticProperties,
+ Predicate<StaticProperty> predicate,
+ Class<T> propertyType) {
+ return staticProperties.stream()
+ .filter(predicate)
+ .filter(propertyType::isInstance)
+ .map(propertyType::cast)
+ .findFirst()
+ .orElseThrow(() -> new IllegalArgumentException("Could not find static property"));
+ }
+
+ private boolean isRuntimeResolvable(StaticProperty staticProperty) {
+ return staticProperty instanceof RuntimeResolvableGroupStaticProperty
+ || staticProperty instanceof RuntimeResolvableOneOfStaticProperty
+ || staticProperty instanceof RuntimeResolvableAnyStaticProperty
+ || staticProperty instanceof RuntimeResolvableTreeInputStaticProperty;
+ }
+}
diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/SinkTesterBase.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/SinkTesterBase.java
new file mode 100644
index 0000000000..dfbbd313be
--- /dev/null
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/SinkTesterBase.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.integration.sinks;
+
+import org.apache.streampipes.extensions.api.pe.IStreamPipesDataSink;
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.graph.DataSinkInvocation;
+import org.apache.streampipes.model.runtime.Event;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+
+import java.util.List;
+
+public abstract class SinkTesterBase implements AutoCloseable {
+
+ private IStreamPipesDataSink sink;
+ private final DataSinkTestConfigurator configurator;
+
+ protected SinkTesterBase() {
+ this.configurator = new DataSinkTestConfigurator();
+ }
+
+ public void run() throws Exception {
+ startSinkService();
+
+ this.sink = createSink();
+ List<SpDataStream> inputStreams = createInputStreams();
+ List<StaticProperty> staticProperties = configureStaticProperties(sink, inputStreams);
+
+ DataSinkInvocation invocation = new DataSinkInvocation(sink.declareConfig().getDescription());
+ invocation.setStaticProperties(staticProperties);
+ invocation.setInputStreams(inputStreams);
+
+ sink.onPipelineStarted(new TestDataSinkParameters(invocation, inputStreams), null);
+
+ List<Event> events = createEvents();
+ for (Event event : events) {
+ sink.onEvent(event);
+ }
+
+ validate(events);
+ }
+
+ protected DataSinkTestConfigurator configurator() {
+ return configurator;
+ }
+
+ protected abstract void startSinkService() throws Exception;
+
+ protected abstract IStreamPipesDataSink createSink() throws Exception;
+
+ protected abstract List<SpDataStream> createInputStreams();
+
+ protected abstract List<StaticProperty> configureStaticProperties(IStreamPipesDataSink sink,
+ List<SpDataStream> inputStreams) throws Exception;
+
+ protected abstract List<Event> createEvents();
+
+ protected abstract void validate(List<Event> events) throws Exception;
+
+ protected void stopSinkService() throws Exception {
+ }
+
+ @Override
+ public void close() throws Exception {
+ Exception failure = null;
+
+ if (sink != null) {
+ try {
+ sink.onPipelineStopped();
+ } catch (Exception e) {
+ failure = e;
+ } finally {
+ sink = null;
+ }
+ }
+
+ try {
+ stopSinkService();
+ } catch (Exception e) {
+ if (failure != null) {
+ failure.addSuppressed(e);
+ } else {
+ failure = e;
+ }
+ }
+
+ if (failure != null) {
+ throw failure;
+ }
+ }
+}
diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/TestDataSinkParameters.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/TestDataSinkParameters.java
new file mode 100644
index 0000000000..b50ab75a9c
--- /dev/null
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/TestDataSinkParameters.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.integration.sinks;
+
+import org.apache.streampipes.extensions.api.extractor.IDataSinkParameterExtractor;
+import org.apache.streampipes.extensions.api.pe.param.IDataSinkParameters;
+import org.apache.streampipes.extensions.api.pe.param.InputStreamParams;
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.graph.DataSinkInvocation;
+import org.apache.streampipes.model.runtime.SchemaInfo;
+import org.apache.streampipes.model.runtime.SourceInfo;
+import org.apache.streampipes.sdk.extractor.DataSinkParameterExtractor;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.IntStream;
+
+public class TestDataSinkParameters implements IDataSinkParameters {
+
+ private final DataSinkInvocation model;
+ private final IDataSinkParameterExtractor extractor;
+ private final List<SourceInfo> inputSourceInfos;
+ private final List<SchemaInfo> inputSchemaInfos;
+ private final Map<String, Integer> sourceIndexes;
+
+ public TestDataSinkParameters(DataSinkInvocation model, List<SpDataStream> inputStreams) {
+ this.model = model;
+ this.extractor = DataSinkParameterExtractor.from(model);
+ this.inputSourceInfos = IntStream.range(0, inputStreams.size())
+ .mapToObj(index -> new SourceInfo("source-" + index, "s" + index))
+ .toList();
+ this.inputSchemaInfos = IntStream.range(0, inputStreams.size())
+ .mapToObj(index -> new SchemaInfo(inputStreams.get(index).getEventSchema(), Collections.emptyList()))
+ .toList();
+ this.sourceIndexes = IntStream.range(0, inputSourceInfos.size())
+ .boxed()
+ .collect(java.util.stream.Collectors.toMap(index -> inputSourceInfos.get(index).getSourceId(), index -> index));
+ }
+
+ @Override
+ public DataSinkInvocation getModel() {
+ return model;
+ }
+
+ @Override
+ public IDataSinkParameterExtractor getExtractor() {
+ return extractor;
+ }
+
+ @Override
+ public IDataSinkParameterExtractor extractor() {
+ return extractor;
+ }
+
+ @Override
+ public List<InputStreamParams> getInputStreamParams() {
+ return List.of();
+ }
+
+ @Override
+ public Map<String, Map<String, Object>> getInEventTypes() {
+ return Map.of();
+ }
+
+ @Override
+ public List<SchemaInfo> getInputSchemaInfos() {
+ return inputSchemaInfos;
+ }
+
+ @Override
+ public List<SourceInfo> getInputSourceInfos() {
+ return inputSourceInfos;
+ }
+
+ @Override
+ public SchemaInfo getInputSchemaInfo(int index) {
+ return inputSchemaInfos.get(index);
+ }
+
+ @Override
+ public SourceInfo getInputSourceInfo(int index) {
+ return inputSourceInfos.get(index);
+ }
+
+ @Override
+ public Integer getSourceIndex(String sourceId) {
+ return sourceIndexes.get(sourceId);
+ }
+}
diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/CamelAzureCosmosDbSinkTester.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/CamelAzureCosmosDbSinkTester.java
new file mode 100644
index 0000000000..fef41a8105
--- /dev/null
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/CamelAzureCosmosDbSinkTester.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.integration.sinks.azure;
+
+import org.apache.streampipes.extensions.api.pe.IStreamPipesDataSink;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.config.KameletSinkStaticPropertyProvider;
+import org.apache.streampipes.integration.containers.AzureCosmosDbEmulatorContainer;
+import org.apache.streampipes.integration.sinks.SinkTesterBase;
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.runtime.Event;
+import org.apache.streampipes.model.runtime.EventFactory;
+import org.apache.streampipes.model.runtime.SchemaInfo;
+import org.apache.streampipes.model.runtime.SourceInfo;
+import org.apache.streampipes.model.schema.EventPropertyPrimitive;
+import org.apache.streampipes.model.schema.EventSchema;
+import org.apache.streampipes.model.staticproperty.CollectionStaticProperty;
+import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty;
+import org.apache.streampipes.model.staticproperty.MappingPropertyUnary;
+import org.apache.streampipes.model.staticproperty.RuntimeResolvableGroupStaticProperty;
+import org.apache.streampipes.model.staticproperty.SecretStaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticPropertyAlternatives;
+import org.apache.streampipes.model.staticproperty.StaticPropertyGroup;
+
+import com.azure.cosmos.CosmosClient;
+import com.azure.cosmos.CosmosClientBuilder;
+import com.azure.cosmos.CosmosContainer;
+import com.azure.cosmos.CosmosDatabase;
+import com.azure.cosmos.models.CosmosContainerProperties;
+import com.azure.cosmos.models.PartitionKey;
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class CamelAzureCosmosDbSinkTester extends SinkTesterBase {
+
+ private static final String TEMPLATE_NAME = "azure-cosmosdb-sink";
+ private static final int COSMOS_STARTUP_ATTEMPTS = 12;
+ private static final long COSMOS_STARTUP_RETRY_DELAY_MS = 5000;
+
+ private final String databaseName = "streampipes-test-" + UUID.randomUUID();
+ private final String containerName = "items";
+ private final List<Map<String, Object>> expectedDocuments = List.of(
+ Map.of("id", "item-1", "value", 42),
+ Map.of("id", "item-2", "value", 84)
+ );
+
+ private AzureCosmosDbEmulatorContainer emulatorContainer;
+ private CosmosDbEmulatorTrustStore trustStore;
+ private CosmosClient cosmosClient;
+ private CosmosContainer verificationContainer;
+ private EventSchema eventSchema;
+ private String emulatorKey;
+
+ @Override
+ protected void startSinkService() throws Exception {
+ emulatorContainer = new AzureCosmosDbEmulatorContainer();
+ emulatorContainer.start();
+
+ trustStore = new CosmosDbEmulatorTrustStore(emulatorContainer.getCertificateEndpoint());
+ trustStore.install();
+ emulatorKey = resolveEmulatorKey();
+
+ cosmosClient = awaitCosmosClientReady();
+
+ cosmosClient.createDatabaseIfNotExists(databaseName);
+ CosmosDatabase database = cosmosClient.getDatabase(databaseName);
+ database.createContainerIfNotExists(new CosmosContainerProperties(containerName, "/id"));
+
+ awaitContainerReady();
+ verificationContainer = cosmosClient.getDatabase(databaseName).getContainer(containerName);
+ }
+
+ @Override
+ protected IStreamPipesDataSink createSink() {
+ var configuration = new TestCamelAzureConnectorsModuleExport()
+ .pipelineElements()
+ .stream()
+ .filter(pipelineElement -> pipelineElement.declareConfig()
+ .getDescription()
+ .getAppId()
+ .endsWith("." + TEMPLATE_NAME))
+ .findFirst()
+ .orElseThrow(() -> new IllegalStateException("Could not find Azure Cosmos DB sink"))
+ .declareConfig();
+
+ return (IStreamPipesDataSink) configuration.getSupplier().get();
+ }
+
+ @Override
+ protected List<SpDataStream> createInputStreams() {
+ this.eventSchema = new EventSchema();
+ this.eventSchema.addEventProperty(
+ new EventPropertyPrimitive("http://www.w3.org/2001/XMLSchema#string", "id", null, null)
+ );
+ this.eventSchema.addEventProperty(
+ new EventPropertyPrimitive("http://www.w3.org/2001/XMLSchema#integer", "value", null, null)
+ );
+
+ SpDataStream inputStream = new SpDataStream();
+ inputStream.setEventSchema(eventSchema);
+ return List.of(inputStream);
+ }
+
+ @Override
+ protected List<StaticProperty> configureStaticProperties(IStreamPipesDataSink sink,
+ List<SpDataStream> inputStreams) throws Exception {
+ List<StaticProperty> staticProperties = configurator().prepareStaticProperties(sink, inputStreams);
+
+ RuntimeResolvableGroupStaticProperty parameterGroup = configurator().requireProperty(
+ staticProperties,
+ KameletSinkStaticPropertyProvider.KAMELET_PARAMETERS_GROUP_KEY,
+ RuntimeResolvableGroupStaticProperty.class
+ );
+ setParameterValue(parameterGroup, "databasename", databaseName);
+ setParameterValue(parameterGroup, "containername", containerName);
+ setParameterValue(parameterGroup, "databaseendpoint", emulatorContainer.getEmulatorEndpoint());
+ setParameterValue(parameterGroup, "accountkey", emulatorKey);
+
+ StaticPropertyGroup messageMappingGroup = configurator().requireProperty(
+ staticProperties,
+ KameletSinkStaticPropertyProvider.MESSAGE_MAPPING_GROUP_KEY,
+ StaticPropertyGroup.class
+ );
+ StaticPropertyAlternatives payloadAlternatives = configurator().requireProperty(
+ messageMappingGroup,
+ KameletSinkStaticPropertyProvider.PAYLOAD_ALTERNATIVES_KEY,
+ StaticPropertyAlternatives.class
+ );
+ configurator().selectAlternative(payloadAlternatives, KameletSinkStaticPropertyProvider.PAYLOAD_EVENT_JSON_KEY);
+
+ CollectionStaticProperty headerMappings = configurator().requireProperty(
+ messageMappingGroup,
+ KameletSinkStaticPropertyProvider.HEADER_MAPPINGS_KEY,
+ CollectionStaticProperty.class
+ );
+ StaticPropertyGroup headerMapping = configurator().addCollectionMember(headerMappings);
+ configurator().requireProperty(
+ headerMapping,
+ KameletSinkStaticPropertyProvider.HEADER_NAME_KEY,
+ FreeTextStaticProperty.class
+ ).setValue("itemPartitionKey");
+ configurator().requireProperty(
+ headerMapping,
+ KameletSinkStaticPropertyProvider.HEADER_FIELD_MAPPING_KEY,
+ MappingPropertyUnary.class
+ ).setSelectedProperty("s0::id");
+
+ return staticProperties;
+ }
+
+ @Override
+ protected List<Event> createEvents() {
+ SourceInfo sourceInfo = new SourceInfo("source-0", "s0");
+ SchemaInfo schemaInfo = new SchemaInfo(eventSchema, List.of());
+
+ return expectedDocuments.stream()
+ .map(document -> EventFactory.fromMap(document, sourceInfo, schemaInfo))
+ .toList();
+ }
+
+ @Override
+ protected void validate(List<Event> events) throws Exception {
+ for (Map<String, Object> expectedDocument : expectedDocuments) {
+ JsonNode item = awaitItem(expectedDocument.get("id").toString());
+ assertNotNull(item);
+ assertEquals(expectedDocument.get("id"), item.get("id").asText());
+ assertEquals(((Number) expectedDocument.get("value")).intValue(), item.get("value").asInt());
+ }
+ }
+
+ @Override
+ protected void stopSinkService() throws Exception {
+ Exception failure = null;
+
+ if (cosmosClient != null) {
+ try {
+ cosmosClient.close();
+ } catch (Exception e) {
+ failure = e;
+ } finally {
+ cosmosClient = null;
+ }
+ }
+
+ if (trustStore != null) {
+ try {
+ trustStore.close();
+ } catch (Exception e) {
+ if (failure != null) {
+ failure.addSuppressed(e);
+ } else {
+ failure = e;
+ }
+ } finally {
+ trustStore = null;
+ }
+ }
+
+ if (emulatorContainer != null) {
+ try {
+ emulatorContainer.stop();
+ } catch (Exception e) {
+ if (failure != null) {
+ failure.addSuppressed(e);
+ } else {
+ failure = e;
+ }
+ } finally {
+ emulatorContainer = null;
+ }
+ }
+
+ if (failure != null) {
+ throw failure;
+ }
+ }
+
+ private void setParameterValue(RuntimeResolvableGroupStaticProperty parameterGroup,
+ String suffix,
+ String value) {
+ StaticProperty property = configurator().requirePropertyMatching(
+ parameterGroup,
+ candidate -> candidate.getInternalName() != null && candidate.getInternalName().endsWith(suffix),
+ StaticProperty.class
+ );
+
+ if (property instanceof FreeTextStaticProperty freeTextStaticProperty) {
+ freeTextStaticProperty.setValue(value);
+ } else if (property instanceof SecretStaticProperty secretStaticProperty) {
+ secretStaticProperty.setValue(value);
+ } else {
+ throw new IllegalArgumentException("Unsupported Kamelet parameter property type: " + property.getClass());
+ }
+ }
+
+ private JsonNode awaitItem(String id) throws Exception {
+ Exception lastException = null;
+
+ for (int attempt = 0; attempt < 20; attempt++) {
+ try {
+ return verificationContainer.readItem(id, new PartitionKey(id), JsonNode.class).getItem();
+ } catch (Exception e) {
+ lastException = e;
+ Thread.sleep(1000);
+ }
+ }
+
+ throw lastException;
+ }
+
+ private void awaitContainerReady() throws Exception {
+ Exception lastException = null;
+
+ for (int attempt = 0; attempt < COSMOS_STARTUP_ATTEMPTS; attempt++) {
+ try (CosmosClient probeClient = createCosmosClient()) {
+ CosmosDatabase probeDatabase = probeClient.getDatabase(databaseName);
+ CosmosContainer probeContainer = probeDatabase.getContainer(containerName);
+ probeDatabase.read();
+ probeContainer.read();
+ return;
+ } catch (Exception e) {
+ lastException = e;
+ Thread.sleep(COSMOS_STARTUP_RETRY_DELAY_MS);
+ }
+ }
+
+ throw lastException;
+ }
+
+ private CosmosClient awaitCosmosClientReady() throws Exception {
+ Exception lastException = null;
+
+ for (int attempt = 0; attempt < COSMOS_STARTUP_ATTEMPTS; attempt++) {
+ try {
+ return createCosmosClient();
+ } catch (Exception e) {
+ lastException = e;
+ Thread.sleep(COSMOS_STARTUP_RETRY_DELAY_MS);
+ }
+ }
+
+ throw lastException;
+ }
+
+ private CosmosClient createCosmosClient() {
+ return new CosmosClientBuilder()
+ .endpoint(emulatorContainer.getEmulatorEndpoint())
+ .key(emulatorKey)
+ .gatewayMode()
+ .contentResponseOnWriteEnabled(true)
+ .buildClient();
+ }
+
+ private String resolveEmulatorKey() throws Exception {
+ String configuredKey = emulatorContainer.getConfiguredEmulatorKey();
+ if (configuredKey != null) {
+ return configuredKey;
+ }
+
+ throw new IllegalStateException(
+ "Could not determine the Cosmos emulator key. "
+ + "Set -Dstreampipes.it.cosmosdb.key or STREAMPIPES_IT_COSMOSDB_KEY."
+ );
+ }
+}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/CamelAzureSinksIntegrationTest.java
similarity index 65%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/CamelAzureSinksIntegrationTest.java
index e01f7adeb1..5df61c022b 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/CamelAzureSinksIntegrationTest.java
@@ -16,17 +16,16 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.integration.sinks.azure;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.junit.jupiter.api.Test;
-import java.util.function.Supplier;
+public class CamelAzureSinksIntegrationTest {
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
-
- NeT getDescription();
-
- Supplier<PeT> getSupplier();
+ @Test
+ public void testAzureCosmosDbSink() throws Exception {
+ try (var tester = new CamelAzureCosmosDbSinkTester()) {
+ tester.run();
+ }
+ }
}
diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/CosmosDbEmulatorTrustStore.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/CosmosDbEmulatorTrustStore.java
new file mode 100644
index 0000000000..32f482dfa0
--- /dev/null
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/CosmosDbEmulatorTrustStore.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.integration.sinks.azure;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import java.io.ByteArrayInputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyStore;
+import java.security.SecureRandom;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+
+public class CosmosDbEmulatorTrustStore implements AutoCloseable {
+
+ private static final char[] TRUST_STORE_PASSWORD = "changeit".toCharArray();
+ private static final String TRUST_STORE_TYPE = "JKS";
+
+ private final String certificateEndpoint;
+ private Path trustStorePath;
+ private String previousTrustStore;
+ private String previousTrustStorePassword;
+ private String previousTrustStoreType;
+
+ public CosmosDbEmulatorTrustStore(String certificateEndpoint) {
+ this.certificateEndpoint = certificateEndpoint;
+ }
+
+ public void install() throws Exception {
+ rememberSystemProperties();
+
+ byte[] pemBytes = downloadCertificate();
+ Certificate certificate = readCertificate(pemBytes);
+
+ KeyStore keyStore = KeyStore.getInstance(TRUST_STORE_TYPE);
+ keyStore.load(null, TRUST_STORE_PASSWORD);
+ keyStore.setCertificateEntry("cosmos-emulator", certificate);
+
+ this.trustStorePath = Files.createTempFile("cosmos-emulator", ".jks");
+ try (var outputStream = Files.newOutputStream(trustStorePath)) {
+ keyStore.store(outputStream, TRUST_STORE_PASSWORD);
+ }
+
+ System.setProperty("javax.net.ssl.trustStore", trustStorePath.toString());
+ System.setProperty("javax.net.ssl.trustStorePassword", new String(TRUST_STORE_PASSWORD));
+ System.setProperty("javax.net.ssl.trustStoreType", TRUST_STORE_TYPE);
+ }
+
+ @Override
+ public void close() throws Exception {
+ restoreSystemProperty("javax.net.ssl.trustStore", previousTrustStore);
+ restoreSystemProperty("javax.net.ssl.trustStorePassword", previousTrustStorePassword);
+ restoreSystemProperty("javax.net.ssl.trustStoreType", previousTrustStoreType);
+
+ if (trustStorePath != null) {
+ Files.deleteIfExists(trustStorePath);
+ trustStorePath = null;
+ }
+ }
+
+ private byte[] downloadCertificate() throws Exception {
+ Exception lastException = null;
+
+ for (int attempt = 0; attempt < 30; attempt++) {
+ try {
+ return fetchCertificate();
+ } catch (Exception e) {
+ lastException = e;
+ Thread.sleep(1000);
+ }
+ }
+
+ throw lastException;
+ }
+
+ private byte[] fetchCertificate() throws Exception {
+ SSLContext sslContext = SSLContext.getInstance("TLS");
+ sslContext.init(null, new TrustManager[] {new TrustAllX509Manager()}, new SecureRandom());
+
+ URL certificateUrl = new URL(certificateEndpoint);
+ int port = certificateUrl.getPort() > 0 ? certificateUrl.getPort() : certificateUrl.getDefaultPort();
+
+ try (SSLSocket socket = (SSLSocket) sslContext.getSocketFactory().createSocket(
+ certificateUrl.getHost(),
+ port
+ )) {
+ socket.setSoTimeout(5000);
+ socket.startHandshake();
+
+ Certificate[] peerCertificates = socket.getSession().getPeerCertificates();
+ if (peerCertificates.length == 0) {
+ throw new IllegalStateException("No peer certificate returned by Cosmos emulator");
+ }
+
+ return ((X509Certificate) peerCertificates[0]).getEncoded();
+ }
+ }
+
+ private Certificate readCertificate(byte[] pemBytes) throws Exception {
+ CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509");
+ return certificateFactory.generateCertificate(new ByteArrayInputStream(pemBytes));
+ }
+
+ private void rememberSystemProperties() {
+ this.previousTrustStore = System.getProperty("javax.net.ssl.trustStore");
+ this.previousTrustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword");
+ this.previousTrustStoreType = System.getProperty("javax.net.ssl.trustStoreType");
+ }
+
+ private void restoreSystemProperty(String key, String value) {
+ if (value == null) {
+ System.clearProperty(key);
+ } else {
+ System.setProperty(key, value);
+ }
+ }
+
+ private static final class TrustAllX509Manager implements X509TrustManager {
+
+ @Override
+ public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType) {
+ }
+
+ @Override
+ public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType) {
+ }
+
+ @Override
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return new java.security.cert.X509Certificate[0];
+ }
+ }
+}
diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/TestAzureCosmosDbApacheCamelKameletSink.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/TestAzureCosmosDbApacheCamelKameletSink.java
new file mode 100644
index 0000000000..2b611e878b
--- /dev/null
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/TestAzureCosmosDbApacheCamelKameletSink.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.integration.sinks.azure;
+
+import org.apache.streampipes.extensions.api.assets.AssetResolver;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.provider.KameletTemplateProvider;
+import org.apache.streampipes.extensions.connectors.camel.sink.ApacheCamelKameletSink;
+
+import com.azure.cosmos.CosmosAsyncClient;
+import com.azure.cosmos.CosmosClientBuilder;
+import org.apache.camel.CamelContext;
+import org.apache.camel.component.azure.cosmosdb.CosmosDbComponent;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.Set;
+
+public class TestAzureCosmosDbApacheCamelKameletSink extends ApacheCamelKameletSink {
+
+ private static final Set<String> LOCAL_HOSTS = Set.of("localhost", "127.0.0.1");
+
+ private CosmosAsyncClient cosmosAsyncClient;
+
+ public TestAzureCosmosDbApacheCamelKameletSink(String appId,
+ String templateName,
+ KameletTemplateProvider templateProvider,
+ AssetResolver assetResolver) {
+ super(appId, templateName, templateProvider, assetResolver);
+ }
+
+ @Override
+ protected ApacheCamelKameletSink newInstance() {
+ return new TestAzureCosmosDbApacheCamelKameletSink(
+ getAppId(),
+ getTemplateName(),
+ getTemplateProvider(),
+ getAssetResolver()
+ );
+ }
+
+ @Override
+ protected void customizeCamelContext(CamelContext context,
+ KameletTemplate template,
+ Map<String, Object> params) {
+ String databaseEndpoint = stringParam(params, "databaseEndpoint");
+ if (!isLocalCosmosEndpoint(databaseEndpoint)) {
+ return;
+ }
+
+ // The Cosmos emulator rejects the direct-mode /addresses lookup, so tests inject a gateway client here.
+ String accountKey = stringParam(params, "accountKey");
+ this.cosmosAsyncClient = new CosmosClientBuilder()
+ .endpoint(databaseEndpoint)
+ .key(accountKey)
+ .gatewayMode()
+ .contentResponseOnWriteEnabled(true)
+ .buildAsyncClient();
+
+ CosmosDbComponent cosmosDbComponent = context.getComponent("azure-cosmosdb", CosmosDbComponent.class);
+ cosmosDbComponent.getConfiguration().setCosmosAsyncClient(cosmosAsyncClient);
+ }
+
+ @Override
+ protected void onSinkStopped() {
+ if (cosmosAsyncClient != null) {
+ cosmosAsyncClient.close();
+ cosmosAsyncClient = null;
+ }
+ }
+
+ private boolean isLocalCosmosEndpoint(String databaseEndpoint) {
+ if (databaseEndpoint == null || databaseEndpoint.isBlank()) {
+ return false;
+ }
+
+ try {
+ URI endpointUri = URI.create(databaseEndpoint);
+ return LOCAL_HOSTS.contains(endpointUri.getHost());
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ }
+
+ private String stringParam(Map<String, Object> params,
+ String key) {
+ Object value = params.get(key);
+ return value == null ? null : String.valueOf(value);
+ }
+}
diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/TestAzureKameletSinkFactory.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/TestAzureKameletSinkFactory.java
new file mode 100644
index 0000000000..5a5a1a8acf
--- /dev/null
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/TestAzureKameletSinkFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.integration.sinks.azure;
+
+import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.assets.KameletSinkAssetResolver;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.factory.KameletSinkFactory;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.model.KameletTemplate;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.provider.KameletTemplateProvider;
+
+public class TestAzureKameletSinkFactory extends KameletSinkFactory {
+
+ private static final String AZURE_COSMOS_DB_SINK = "azure-cosmosdb-sink";
+
+ public TestAzureKameletSinkFactory(KameletTemplateProvider templateProvider,
+ String appIdPrefix,
+ String assetRoot) {
+ super(templateProvider, appIdPrefix, assetRoot);
+ }
+
+ @Override
+ protected IStreamPipesPipelineElement<?> createSink(KameletTemplate template) {
+ if (!AZURE_COSMOS_DB_SINK.equals(template.name())) {
+ return super.createSink(template);
+ }
+
+ String appId = makeAppId(template.name());
+ return new TestAzureCosmosDbApacheCamelKameletSink(
+ appId,
+ template.name(),
+ getTemplateProvider(),
+ new KameletSinkAssetResolver(appId, template.name(), getAssetRoot(), getTemplateProvider())
+ );
+ }
+}
diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/TestCamelAzureConnectorsModuleExport.java
similarity index 55%
copy from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
copy to streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/TestCamelAzureConnectorsModuleExport.java
index e01f7adeb1..7de7e1af7e 100644
--- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/config/IPipelineElementConfiguration.java
+++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/sinks/azure/TestCamelAzureConnectorsModuleExport.java
@@ -16,17 +16,16 @@
*
*/
-package org.apache.streampipes.extensions.api.pe.config;
+package org.apache.streampipes.integration.sinks.azure;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.extensions.connectors.camel.azure.CamelAzureConnectorsModuleExport;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.factory.KameletSinkFactory;
+import org.apache.streampipes.extensions.connectors.camel.kamelet.provider.KameletTemplateProvider;
-import java.util.function.Supplier;
+public class TestCamelAzureConnectorsModuleExport extends CamelAzureConnectorsModuleExport {
-public interface IPipelineElementConfiguration<NeT extends NamedStreamPipesEntity,
- PeT extends IStreamPipesPipelineElement<?>> {
-
- NeT getDescription();
-
- Supplier<PeT> getSupplier();
+ @Override
+ protected KameletSinkFactory createSinkFactory(KameletTemplateProvider templateProvider) {
+ return new TestAzureKameletSinkFactory(templateProvider, getAppIdPrefix(), getAssetRoot());
+ }
}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java
index 1fa3c01a42..b7ee572bd3 100644
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java
@@ -73,8 +73,7 @@ public class ExtensionsFinder {
.stream()
.map(IStreamPipesPipelineElement::declareConfig)
.filter(configType::isInstance)
- .map(config -> new AssetModel(config.getDescription()
- .getAppId(), peType))
+ .map(config -> new AssetModel(config.getDescription().getAppId(), peType, config.getAssetResolver()))
.toList();
}
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/secret/SecretVisitor.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/secret/SecretVisitor.java
index 12f744f9d2..04cd621b93 100644
--- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/secret/SecretVisitor.java
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/secret/SecretVisitor.java
@@ -36,6 +36,8 @@ import org.apache.streampipes.model.staticproperty.StaticPropertyAlternatives;
import org.apache.streampipes.model.staticproperty.StaticPropertyGroup;
import org.apache.streampipes.model.staticproperty.StaticPropertyVisitor;
+import java.util.Objects;
+
public class SecretVisitor implements StaticPropertyVisitor {
private ISecretHandler secretHandler;
@@ -96,7 +98,7 @@ public class SecretVisitor implements StaticPropertyVisitor {
@Override
public void visit(SecretStaticProperty secretStaticProperty) {
- if (secretHandler.shouldApply(secretStaticProperty.getEncrypted())) {
+ if (Objects.nonNull(secretStaticProperty.getEncrypted()) && secretHandler.shouldApply(secretStaticProperty.getEncrypted())) {
String newValue = secretHandler.apply(secretStaticProperty.getValue());
secretStaticProperty.setValue(newValue);
secretStaticProperty.setEncrypted(!secretStaticProperty.getEncrypted());
diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/AbstractPipelineElementResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/AbstractPipelineElementResource.java
index b16f23b023..e14a778ba6 100644
--- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/AbstractPipelineElementResource.java
+++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/AbstractPipelineElementResource.java
@@ -19,6 +19,8 @@
package org.apache.streampipes.rest.extensions;
import org.apache.streampipes.commons.constants.GlobalStreamPipesConstants;
+import org.apache.streampipes.commons.media.ImageMimeTypeDetector;
+import org.apache.streampipes.extensions.api.assets.AssetResolver;
import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
import org.apache.streampipes.extensions.management.assets.AssetZipGenerator;
import org.apache.streampipes.extensions.management.init.DeclarersSingleton;
@@ -29,7 +31,6 @@ import org.apache.streampipes.model.grounding.EventGrounding;
import org.apache.streampipes.model.grounding.TransportProtocol;
import com.google.common.base.Charsets;
-import com.google.common.io.Resources;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +40,6 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import java.io.IOException;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -49,8 +49,6 @@ public abstract class AbstractPipelineElementResource<
T extends IStreamPipesPipelineElement<?>>
extends AbstractExtensionsResource {
- private static final String SLASH = "/";
-
private static final Logger LOG = LoggerFactory.getLogger(AbstractPipelineElementResource.class);
@GetMapping(path = "{appId}", produces = MediaType.APPLICATION_JSON_VALUE)
@@ -60,21 +58,25 @@ public abstract class AbstractPipelineElementResource<
@GetMapping(path = "{appId}/assets", produces = "application/zip")
public ResponseEntity<?> getAssets(@PathVariable("appId") String appId) {
- List<String> includedAssets = getDeclarerById(appId).declareConfig().getDescription().getIncludedAssets();
+ var config = getDeclarerById(appId).declareConfig();
+ List<String> includedAssets = config.getDescription().getIncludedAssets();
try {
- return ok(new AssetZipGenerator(appId, includedAssets).makeZip());
+ return ok(new AssetZipGenerator(includedAssets, config.getAssetResolver()).makeZip());
} catch (IOException e) {
e.printStackTrace();
return serverError();
}
}
- @GetMapping(path = "{appId}/assets/icon", produces = MediaType.IMAGE_PNG_VALUE)
+ @GetMapping(path = "{appId}/assets/icon")
public ResponseEntity<byte[]> getIconAsset(@PathVariable("appId") String appId) throws IOException {
try {
- URL iconUrl = Resources.getResource(makeIconPath(appId));
- return ok(Resources.toByteArray(iconUrl));
- } catch (IllegalArgumentException e) {
+ byte[] icon = getDeclarerById(appId).declareConfig().getAssetResolver().getAsset(GlobalStreamPipesConstants
+ .STD_ICON_NAME);
+ return ResponseEntity.ok()
+ .contentType(MediaType.parseMediaType(ImageMimeTypeDetector.detect(icon)))
+ .body(icon);
+ } catch (IOException e) {
LOG.warn("No icon resource found for pipeline element {}", appId);
return ResponseEntity.status(HttpStatus.SC_BAD_REQUEST).build();
}
@@ -83,20 +85,24 @@ public abstract class AbstractPipelineElementResource<
@GetMapping(path = "{id}/assets/documentation", produces = MediaType.TEXT_PLAIN_VALUE)
public ResponseEntity<String> getDocumentationAsset(@PathVariable("id") String elementId) throws IOException {
try {
- URL documentationUrl = Resources.getResource(makeDocumentationPath(elementId));
- return ok(Resources.toString(documentationUrl, Charsets.UTF_8));
- } catch (IllegalArgumentException e) {
+ return ok(new String(
+ getDeclarerById(elementId).declareConfig().getAssetResolver().getAsset(
+ GlobalStreamPipesConstants.STD_DOCUMENTATION_NAME),
+ Charsets.UTF_8
+ ));
+ } catch (IOException e) {
LOG.warn("No documentation resource found for pipeline element {}", elementId);
return ResponseEntity.status(HttpStatus.SC_BAD_REQUEST).build();
}
}
protected NamedStreamPipesEntity prepareElement(String appId) {
- return rewrite(getById(appId));
+ var config = getDeclarerById(appId).declareConfig();
+ return rewrite(config.getDescription(), config.getAssetResolver());
}
protected NamedStreamPipesEntity prepareElement(NamedStreamPipesEntity desc) {
- return rewrite(desc);
+ return rewrite(desc, null);
}
protected T getDeclarerById(String appId) {
@@ -109,13 +115,20 @@ public abstract class AbstractPipelineElementResource<
}
protected NamedStreamPipesEntity rewrite(NamedStreamPipesEntity desc) {
+ return rewrite(desc, null);
+ }
+
+ protected NamedStreamPipesEntity rewrite(NamedStreamPipesEntity desc,
+ AssetResolver assetResolver) {
//TODO remove this and find a better solution
if (desc != null) {
// TODO remove after full internationalization support has been implemented
if (desc.isIncludesLocales()) {
try {
- desc = new LabelGenerator<>(desc).generateLabels();
+ desc = assetResolver == null
+ ? new LabelGenerator<>(desc).generateLabels()
+ : new LabelGenerator<>(desc, true, assetResolver).generateLabels();
} catch (IOException e) {
e.printStackTrace();
}
@@ -143,17 +156,5 @@ public abstract class AbstractPipelineElementResource<
return grounding;
}
- private String makeIconPath(String appId) {
- return makePath(appId, GlobalStreamPipesConstants.STD_ICON_NAME);
- }
-
- private String makeDocumentationPath(String appId) {
- return makePath(appId, GlobalStreamPipesConstants.STD_DOCUMENTATION_NAME);
- }
-
- private String makePath(String appId, String assetAppendix) {
- return appId + SLASH + assetAppendix;
- }
-
protected abstract Map<String, T> getElementDeclarers();
}
diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/connect/AdapterAssetResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/connect/AdapterAssetResource.java
index c64611dad5..38627bd9ff 100644
--- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/connect/AdapterAssetResource.java
+++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/connect/AdapterAssetResource.java
@@ -19,6 +19,7 @@
package org.apache.streampipes.rest.extensions.connect;
import org.apache.streampipes.commons.constants.GlobalStreamPipesConstants;
+import org.apache.streampipes.commons.media.ImageMimeTypeDetector;
import org.apache.streampipes.extensions.management.assets.AssetZipGenerator;
import org.apache.streampipes.extensions.management.connect.ConnectWorkerDescriptionProvider;
import org.apache.streampipes.model.message.Notifications;
@@ -66,11 +67,14 @@ public class AdapterAssetResource extends AbstractSharedRestInterface {
}
- @GetMapping(path = "/{id}/assets/icon", produces = MediaType.IMAGE_PNG_VALUE)
+ @GetMapping(path = "/{id}/assets/icon")
public ResponseEntity<byte[]> getIconAsset(@PathVariable("id") String elementId) throws IOException {
var adapterConfig = this.connectWorkerDescriptionProvider.getAdapterConfiguration(elementId);
if (adapterConfig.isPresent()) {
- return ok(adapterConfig.get().getAssetResolver().getAsset(GlobalStreamPipesConstants.STD_ICON_NAME));
+ byte[] icon = adapterConfig.get().getAssetResolver().getAsset(GlobalStreamPipesConstants.STD_ICON_NAME);
+ return ResponseEntity.ok()
+ .contentType(MediaType.parseMediaType(ImageMimeTypeDetector.detect(icon)))
+ .body(icon);
} else {
throw new IOException("Could not find adapter");
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementAsset.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementAsset.java
index 5eb9bf6b78..af8510551a 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementAsset.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementAsset.java
@@ -17,6 +17,7 @@
*/
package org.apache.streampipes.rest.impl;
+import org.apache.streampipes.commons.media.ImageMimeTypeDetector;
import org.apache.streampipes.manager.assets.AssetManager;
import org.apache.streampipes.rest.core.base.impl.AbstractRestResource;
import org.apache.streampipes.storage.management.StorageDispatcher;
@@ -38,10 +39,13 @@ public class PipelineElementAsset extends AbstractRestResource {
private static final Logger LOG = LoggerFactory.getLogger(PipelineElementAsset.class);
- @GetMapping(path = "/{appId}/assets/icon", produces = MediaType.IMAGE_PNG_VALUE)
+ @GetMapping(path = "/{appId}/assets/icon")
public ResponseEntity<?> getIconAsset(@PathVariable("appId") String appId) {
try {
- return ok(AssetManager.getAssetIcon(appId));
+ byte[] icon = AssetManager.getAssetIcon(appId);
+ return ResponseEntity.ok()
+ .contentType(MediaType.parseMediaType(ImageMimeTypeDetector.detect(icon)))
+ .body(icon);
} catch (IOException e) {
return fail();
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java
index 15ac3f035f..ac2fe8e376 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.rest.impl.admin;
+import org.apache.streampipes.commons.media.ImageMimeTypeDetector;
import org.apache.streampipes.manager.assets.AssetManager;
import org.apache.streampipes.manager.extensions.AvailableExtensionsProvider;
import org.apache.streampipes.manager.extensions.ExtensionsResourceUrlProvider;
@@ -52,11 +53,13 @@ public class ExtensionsServiceEndpointResource extends AbstractAuthGuardedRestRe
return ok(allExtensions);
}
- @PostMapping(path = "/icon", produces = "image/png")
+ @PostMapping(path = "/icon")
public ResponseEntity<byte[]> getExtensionItemIcon(@RequestBody ExtensionItemDescription endpointItem) {
try {
byte[] imageBytes = getIconImage(endpointItem);
- return ok(imageBytes);
+ return ResponseEntity.ok()
+ .contentType(MediaType.parseMediaType(ImageMimeTypeDetector.detect(imageBytes)))
+ .body(imageBytes);
} catch (IOException e) {
throw new SpMessageException(HttpStatus.BAD_REQUEST, e);
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java
index e9b81ab80c..5d22800231 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java
@@ -21,6 +21,7 @@ package org.apache.streampipes.rest.impl.connect;
import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException;
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.commons.exceptions.connect.AdapterException;
+import org.apache.streampipes.commons.media.ImageMimeTypeDetector;
import org.apache.streampipes.connect.management.management.DescriptionManagement;
import org.apache.streampipes.manager.api.extensions.IExtensionsServiceEndpointGenerator;
import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator;
@@ -84,7 +85,7 @@ public class DescriptionResource extends AbstractAdapterResource<DescriptionMana
}
}
- @GetMapping(path = "/{id}/assets/icon", produces = "image/png")
+ @GetMapping(path = "/{id}/assets/icon")
@PreAuthorize("this.hasReadAuthority()")
public ResponseEntity<?> getAdapterIconAsset(@PathVariable("id") String id) {
try {
@@ -103,7 +104,9 @@ public class DescriptionResource extends AbstractAdapterResource<DescriptionMana
LOG.error("Not found adapter with id {}", id);
return fail();
} else {
- return ok(result);
+ return ResponseEntity.ok()
+ .contentType(MediaType.parseMediaType(ImageMimeTypeDetector.detect(result)))
+ .body(result);
}
} catch (AdapterException | NoServiceEndpointsAvailableException e) {
LOG.error("Not found adapter with id {}", id, e);
diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/processor/DataProcessorConfiguration.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/processor/DataProcessorConfiguration.java
index cd8e5d24b4..0636b68944 100644
--- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/processor/DataProcessorConfiguration.java
+++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/processor/DataProcessorConfiguration.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.sdk.builder.processor;
+import org.apache.streampipes.extensions.api.assets.AssetResolver;
import org.apache.streampipes.extensions.api.pe.IStreamPipesDataProcessor;
import org.apache.streampipes.extensions.api.pe.config.IDataProcessorConfiguration;
import org.apache.streampipes.model.graph.DataProcessorDescription;
@@ -28,16 +29,25 @@ public class DataProcessorConfiguration implements IDataProcessorConfiguration {
private final Supplier<IStreamPipesDataProcessor> supplier;
private final DataProcessorDescription dataProcessorDescription;
+ private final AssetResolver assetResolver;
public static DataProcessorConfiguration create(Supplier<IStreamPipesDataProcessor> supplier,
DataProcessorDescription dataProcessorDescription) {
- return new DataProcessorConfiguration(supplier, dataProcessorDescription);
+ return new DataProcessorConfiguration(supplier, dataProcessorDescription, null);
+ }
+
+ public static DataProcessorConfiguration create(Supplier<IStreamPipesDataProcessor> supplier,
+ DataProcessorDescription dataProcessorDescription,
+ AssetResolver assetResolver) {
+ return new DataProcessorConfiguration(supplier, dataProcessorDescription, assetResolver);
}
private DataProcessorConfiguration(Supplier<IStreamPipesDataProcessor> supplier,
- DataProcessorDescription dataProcessorDescription) {
+ DataProcessorDescription dataProcessorDescription,
+ AssetResolver assetResolver) {
this.supplier = supplier;
this.dataProcessorDescription = dataProcessorDescription;
+ this.assetResolver = assetResolver;
}
@Override
@@ -49,4 +59,9 @@ public class DataProcessorConfiguration implements IDataProcessorConfiguration {
public Supplier<IStreamPipesDataProcessor> getSupplier() {
return supplier;
}
+
+ @Override
+ public AssetResolver getAssetResolver() {
+ return assetResolver == null ? IDataProcessorConfiguration.super.getAssetResolver() : assetResolver;
+ }
}
diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/sink/DataSinkConfiguration.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/sink/DataSinkConfiguration.java
index ae9c3beeb9..deacb36b78 100644
--- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/sink/DataSinkConfiguration.java
+++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/sink/DataSinkConfiguration.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.sdk.builder.sink;
+import org.apache.streampipes.extensions.api.assets.AssetResolver;
import org.apache.streampipes.extensions.api.pe.IStreamPipesDataSink;
import org.apache.streampipes.extensions.api.pe.config.IDataSinkConfiguration;
import org.apache.streampipes.model.graph.DataSinkDescription;
@@ -28,16 +29,25 @@ public class DataSinkConfiguration implements IDataSinkConfiguration {
private final Supplier<IStreamPipesDataSink> supplier;
private final DataSinkDescription dataSinkDescription;
+ private final AssetResolver assetResolver;
public static DataSinkConfiguration create(Supplier<IStreamPipesDataSink> supplier,
DataSinkDescription dataSinkDescription) {
- return new DataSinkConfiguration(supplier, dataSinkDescription);
+ return new DataSinkConfiguration(supplier, dataSinkDescription, null);
+ }
+
+ public static DataSinkConfiguration create(Supplier<IStreamPipesDataSink> supplier,
+ DataSinkDescription dataSinkDescription,
+ AssetResolver assetResolver) {
+ return new DataSinkConfiguration(supplier, dataSinkDescription, assetResolver);
}
private DataSinkConfiguration(Supplier<IStreamPipesDataSink> supplier,
- DataSinkDescription dataSinkDescription) {
+ DataSinkDescription dataSinkDescription,
+ AssetResolver assetResolver) {
this.supplier = supplier;
this.dataSinkDescription = dataSinkDescription;
+ this.assetResolver = assetResolver;
}
@Override
@@ -49,4 +59,9 @@ public class DataSinkConfiguration implements IDataSinkConfiguration {
public Supplier<IStreamPipesDataSink> getSupplier() {
return supplier;
}
+
+ @Override
+ public AssetResolver getAssetResolver() {
+ return assetResolver == null ? IDataSinkConfiguration.super.getAssetResolver() : assetResolver;
+ }
}
diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/stream/DataStreamConfiguration.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/stream/DataStreamConfiguration.java
index 5b42770fa2..c1adee002f 100644
--- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/stream/DataStreamConfiguration.java
+++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/stream/DataStreamConfiguration.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.sdk.builder.stream;
+import org.apache.streampipes.extensions.api.assets.AssetResolver;
import org.apache.streampipes.extensions.api.pe.IStreamPipesDataStream;
import org.apache.streampipes.extensions.api.pe.config.IDataStreamConfiguration;
import org.apache.streampipes.model.SpDataStream;
@@ -28,16 +29,25 @@ public class DataStreamConfiguration implements IDataStreamConfiguration {
private final Supplier<IStreamPipesDataStream> supplier;
private final SpDataStream dataStream;
+ private final AssetResolver assetResolver;
public static DataStreamConfiguration create(Supplier<IStreamPipesDataStream> supplier,
SpDataStream dataStream) {
- return new DataStreamConfiguration(supplier, dataStream);
+ return new DataStreamConfiguration(supplier, dataStream, null);
+ }
+
+ public static DataStreamConfiguration create(Supplier<IStreamPipesDataStream> supplier,
+ SpDataStream dataStream,
+ AssetResolver assetResolver) {
+ return new DataStreamConfiguration(supplier, dataStream, assetResolver);
}
public DataStreamConfiguration(Supplier<IStreamPipesDataStream> supplier,
- SpDataStream dataStream) {
+ SpDataStream dataStream,
+ AssetResolver assetResolver) {
this.supplier = supplier;
this.dataStream = dataStream;
+ this.assetResolver = assetResolver;
}
@Override
@@ -49,4 +59,9 @@ public class DataStreamConfiguration implements IDataStreamConfiguration {
public Supplier<IStreamPipesDataStream> getSupplier() {
return supplier;
}
+
+ @Override
+ public AssetResolver getAssetResolver() {
+ return assetResolver == null ? IDataStreamConfiguration.super.getAssetResolver() : assetResolver;
+ }
}
diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionItemProvider.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionItemProvider.java
index c4098a371b..e444f3198a 100644
--- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionItemProvider.java
+++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionItemProvider.java
@@ -19,7 +19,6 @@
package org.apache.streampipes.service.extensions;
import org.apache.streampipes.extensions.api.assets.AssetResolver;
-import org.apache.streampipes.extensions.api.assets.DefaultAssetResolver;
import org.apache.streampipes.extensions.api.connect.IAdapterConfiguration;
import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter;
import org.apache.streampipes.extensions.management.init.DeclarersSingleton;
@@ -57,9 +56,9 @@ public class ExtensionItemProvider {
return
DeclarersSingleton.getInstance().getDeclarers().values()
.stream()
- .map(declarer -> declarer.declareConfig().getDescription())
- .peek(entity -> applyLocales(entity, new DefaultAssetResolver(entity.getAppId())))
- .map(e -> (NamedStreamPipesEntity) e);
+ .map(declarer -> declarer.declareConfig())
+ .peek(config -> applyLocales(config.getDescription(), config.getAssetResolver()))
+ .map(config -> (NamedStreamPipesEntity) config.getDescription());
}
private Stream<NamedStreamPipesEntity> getAdapterDescriptions() {
diff --git a/ui/projects/streampipes/shared-ui/src/lib/components/pipeline-element/pipeline-element.component.ts b/ui/projects/streampipes/shared-ui/src/lib/components/pipeline-element/pipeline-element.component.ts
index 9ed0023f53..fc669efcde 100644
--- a/ui/projects/streampipes/shared-ui/src/lib/components/pipeline-element/pipeline-element.component.ts
+++ b/ui/projects/streampipes/shared-ui/src/lib/components/pipeline-element/pipeline-element.component.ts
@@ -61,7 +61,9 @@ export class PipelineElementComponent {
checkImageAvailable() {
if (
this.pipelineElement.includesAssets &&
- this.pipelineElement.includedAssets.indexOf('icon.png') > -1
+ this.pipelineElement.includedAssets?.some(asset =>
+ asset.startsWith('icon.'),
+ )
) {
this.image = this.sanitizer.bypassSecurityTrustUrl(
this.makeAssetIconUrl(),
diff --git a/ui/src/app/connect/components/existing-adapters/existing-adapters.component.ts b/ui/src/app/connect/components/existing-adapters/existing-adapters.component.ts
index bd0e80bc78..ed68dc74f7 100644
--- a/ui/src/app/connect/components/existing-adapters/existing-adapters.component.ts
+++ b/ui/src/app/connect/components/existing-adapters/existing-adapters.component.ts
@@ -288,7 +288,7 @@ export class ExistingAdaptersComponent implements OnInit, OnDestroy {
}
getIconUrl(adapter: AdapterDescription) {
- if (adapter.includedAssets.length > 0) {
+ if (adapter.includedAssets?.some(asset => asset.startsWith('icon.'))) {
return (
this.pipelineElementAssetService.getAssetUrl(adapter.appId) +
'/icon'
diff --git a/ui/src/app/core-ui/static-properties/static-free-input/static-free-input.component.ts b/ui/src/app/core-ui/static-properties/static-free-input/static-free-input.component.ts
index 97d02656fb..09ac9adb6f 100644
--- a/ui/src/app/core-ui/static-properties/static-free-input/static-free-input.component.ts
+++ b/ui/src/app/core-ui/static-properties/static-free-input/static-free-input.component.ts
@@ -100,27 +100,32 @@ export class StaticFreeInputComponent
const validators: ValidatorFn[] = [];
if (!this.staticProperty.optional) {
validators.push(Validators.required);
- }
- if (
- DataType.isNumberType(this.staticProperty.requiredDatatype) ||
- DataType.isNumberType(this.staticProperty.requiredDomainProperty)
- ) {
- validators.push(ValidateNumber);
- this.errorMessage = this.translateService.instant(
- 'The value should be a number',
- );
- } else if (
- this.staticProperty.requiredDomainProperty === SemanticType.SO_URL
- ) {
- validators.push(ValidateUrl);
- this.errorMessage = this.translateService.instant(
- 'Please enter a valid URL',
- );
- } else if (this.staticProperty.requiredDatatype === DataType.STRING) {
- validators.push(ValidateString);
- this.errorMessage = this.translateService.instant(
- 'Please enter a valid String',
- );
+ if (
+ DataType.isNumberType(this.staticProperty.requiredDatatype) ||
+ DataType.isNumberType(
+ this.staticProperty.requiredDomainProperty,
+ )
+ ) {
+ validators.push(ValidateNumber);
+ this.errorMessage = this.translateService.instant(
+ 'The value should be a number',
+ );
+ } else if (
+ this.staticProperty.requiredDomainProperty ===
+ SemanticType.SO_URL
+ ) {
+ validators.push(ValidateUrl);
+ this.errorMessage = this.translateService.instant(
+ 'Please enter a valid URL',
+ );
+ } else if (
+ this.staticProperty.requiredDatatype === DataType.STRING
+ ) {
+ validators.push(ValidateString);
+ this.errorMessage = this.translateService.instant(
+ 'Please enter a valid String',
+ );
+ }
}
return validators;