You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2021/07/21 10:43:09 UTC

[incubator-streampipes-extensions] branch STREAMPIPES-319 updated: [STREAMPIPES-319] Refactor flink config

This is an automated email from the ASF dual-hosted git repository.

riemer pushed a commit to branch STREAMPIPES-319
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes-extensions.git


The following commit(s) were added to refs/heads/STREAMPIPES-319 by this push:
     new f4f27c3  [STREAMPIPES-319] Refactor flink config
f4f27c3 is described below

commit f4f27c39d888ae9e165c9b24ef79d63603ab2a2b
Author: Dominik Riemer <ri...@fzi.de>
AuthorDate: Wed Jul 21 12:42:54 2021 +0200

    [STREAMPIPES-319] Refactor flink config
---
 .../aggregation/flink/config/AggregationFlinkConfig.java          | 3 ++-
 .../flink/processor/aggregation/AggregationController.java        | 6 +++++-
 .../aggregation/flink/processor/count/CountController.java        | 6 +++++-
 .../flink/processor/eventcount/EventCountController.java          | 7 ++++++-
 .../aggregation/flink/processor/rate/EventRateController.java     | 8 ++++++--
 .../processors/enricher/flink/config/EnricherFlinkConfig.java     | 4 +++-
 .../detection/flink/config/PatternDetectionFlinkConfig.java       | 3 ++-
 .../processors/statistics/flink/config/StatisticsFlinkConfig.java | 3 ++-
 .../transformation/flink/config/TransformationFlinkConfig.java    | 3 ++-
 .../sinks/databases/flink/config/DatabasesFlinkConfig.java        | 3 ++-
 10 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java
index 134a6cf..c925418 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java
@@ -18,6 +18,7 @@
 
 package org.apache.streampipes.processors.aggregation.flink.config;
 
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
 import org.apache.streampipes.svcdiscovery.api.SpConfig;
 import org.apache.streampipes.container.model.PeConfig;
 
@@ -33,7 +34,7 @@ public enum AggregationFlinkConfig implements PeConfig {
 
 
   AggregationFlinkConfig() {
-    config = SpConfig.getSpConfig(service_id);
+    config = SpServiceDiscovery.getSpConfig(service_id);
 
     config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
     config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java
index 9d18106..d8baeca 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java
@@ -19,8 +19,10 @@
 package org.apache.streampipes.processors.aggregation.flink.processor.aggregation;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.streampipes.client.StreamPipesClient;
 import org.apache.streampipes.commons.exceptions.SpRuntimeException;
 import org.apache.streampipes.container.api.ResolvesContainerProvidedOutputStrategy;
+import org.apache.streampipes.container.config.ConfigExtractor;
 import org.apache.streampipes.model.DataProcessorType;
 import org.apache.streampipes.model.graph.DataProcessorDescription;
 import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -95,7 +97,9 @@ public class AggregationController extends FlinkDataProcessorDeclarer<Aggregatio
 
   @Override
   public FlinkDataProcessorRuntime<AggregationParameters> getRuntime(DataProcessorInvocation graph,
-                                                                     ProcessingElementParameterExtractor extractor) {
+                                                                     ProcessingElementParameterExtractor extractor,
+                                                                     ConfigExtractor configExtractor,
+                                                                     StreamPipesClient streamPipesClient) {
 
     List<String> groupBy = extractor.mappingPropertyValues("groupBy");
 
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/count/CountController.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/count/CountController.java
index d635aed..1e757b1 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/count/CountController.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/count/CountController.java
@@ -18,6 +18,8 @@
 
 package org.apache.streampipes.processors.aggregation.flink.processor.count;
 
+import org.apache.streampipes.client.StreamPipesClient;
+import org.apache.streampipes.container.config.ConfigExtractor;
 import org.apache.streampipes.model.DataProcessorType;
 import org.apache.streampipes.model.graph.DataProcessorDescription;
 import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -72,7 +74,9 @@ public class CountController extends FlinkDataProcessorDeclarer<CountParameters>
 
   @Override
   public FlinkDataProcessorRuntime<CountParameters> getRuntime(DataProcessorInvocation graph,
-                                                               ProcessingElementParameterExtractor extractor) {
+                                                               ProcessingElementParameterExtractor extractor,
+                                                               ConfigExtractor configExtractor,
+                                                               StreamPipesClient streamPipesClient) {
 
     Integer timeWindowSize = extractor.singleValueParameter(TIME_WINDOW_KEY, Integer.class);
     String scale = extractor.selectedSingleValueInternalName(SCALE_KEY, String.class);
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/eventcount/EventCountController.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/eventcount/EventCountController.java
index 4700330..b5f3e60 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/eventcount/EventCountController.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/eventcount/EventCountController.java
@@ -17,6 +17,8 @@
  */
 package org.apache.streampipes.processors.aggregation.flink.processor.eventcount;
 
+import org.apache.streampipes.client.StreamPipesClient;
+import org.apache.streampipes.container.config.ConfigExtractor;
 import org.apache.streampipes.model.DataProcessorType;
 import org.apache.streampipes.model.graph.DataProcessorDescription;
 import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -67,7 +69,10 @@ public class EventCountController extends FlinkDataProcessorDeclarer<EventCountP
   }
 
   @Override
-  public FlinkDataProcessorRuntime<EventCountParameters> getRuntime(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+  public FlinkDataProcessorRuntime<EventCountParameters> getRuntime(DataProcessorInvocation graph,
+                                                                    ProcessingElementParameterExtractor extractor,
+                                                                    ConfigExtractor configExtractor,
+                                                                    StreamPipesClient streamPipesClient) {
     Integer timeWindowSize = extractor.singleValueParameter(TIME_WINDOW_KEY, Integer.class);
     String scale = extractor.selectedSingleValueInternalName(SCALE_KEY, String.class);
     EventCountParameters staticParam = new EventCountParameters(graph, timeWindowSize, scale);
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java
index 7522289..d58d76c 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java
@@ -18,6 +18,8 @@
 
 package org.apache.streampipes.processors.aggregation.flink.processor.rate;
 
+import org.apache.streampipes.client.StreamPipesClient;
+import org.apache.streampipes.container.config.ConfigExtractor;
 import org.apache.streampipes.model.DataProcessorType;
 import org.apache.streampipes.model.graph.DataProcessorDescription;
 import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -57,7 +59,9 @@ public class EventRateController extends FlinkDataProcessorDeclarer<EventRatePar
 
   @Override
   public FlinkDataProcessorRuntime<EventRateParameter> getRuntime(DataProcessorInvocation graph,
-                                                                  ProcessingElementParameterExtractor extractor) {
+                                                                  ProcessingElementParameterExtractor extractor,
+                                                                  ConfigExtractor configExtractor,
+                                                                  StreamPipesClient streamPipesClient) {
     Integer avgRate = extractor.singleValueParameter(RATE_KEY, Integer.class);
 
     EventRateParameter staticParam = new EventRateParameter(graph, avgRate);
@@ -65,4 +69,4 @@ public class EventRateController extends FlinkDataProcessorDeclarer<EventRatePar
     return new EventRateProgram(staticParam, AggregationFlinkConfig.INSTANCE.getDebug());
 
   }
-}
\ No newline at end of file
+}
diff --git a/streampipes-processors-enricher-flink/src/main/java/org/apache/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java b/streampipes-processors-enricher-flink/src/main/java/org/apache/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java
index 909c187..a1b3bd5 100644
--- a/streampipes-processors-enricher-flink/src/main/java/org/apache/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java
+++ b/streampipes-processors-enricher-flink/src/main/java/org/apache/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java
@@ -18,6 +18,7 @@
 
 package org.apache.streampipes.processors.enricher.flink.config;
 
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
 import org.apache.streampipes.svcdiscovery.api.SpConfig;
 import org.apache.streampipes.container.model.PeConfig;
 
@@ -30,8 +31,9 @@ public enum EnricherFlinkConfig implements PeConfig {
   private final static String service_id = "pe/org.apache.streampipes.processors.enricher.flink";
   private final static String service_name = "Processors Enricher Flink";
   private final static String service_container_name = "processors-enricher-flink";
+
   EnricherFlinkConfig() {
-    config = SpConfig.getSpConfig(service_id);
+    config = SpServiceDiscovery.getSpConfig(service_id);
 
     config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
     config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/apache/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/apache/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java
index fd12d75..47ffb22 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/apache/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/apache/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java
@@ -18,6 +18,7 @@
 
 package org.apache.streampipes.processors.pattern.detection.flink.config;
 
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
 import org.apache.streampipes.svcdiscovery.api.SpConfig;
 import org.apache.streampipes.container.model.PeConfig;
 
@@ -32,7 +33,7 @@ public enum PatternDetectionFlinkConfig implements PeConfig {
   private final static String service_container_name = "processors-pattern-detection-flink";
 
   PatternDetectionFlinkConfig() {
-    config = SpConfig.getSpConfig(service_id);
+    config = SpServiceDiscovery.getSpConfig(service_id);
 
     config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
     config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/apache/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java b/streampipes-processors-statistics-flink/src/main/java/org/apache/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java
index da31662..d240b4e 100644
--- a/streampipes-processors-statistics-flink/src/main/java/org/apache/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java
+++ b/streampipes-processors-statistics-flink/src/main/java/org/apache/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java
@@ -18,6 +18,7 @@
 
 package org.apache.streampipes.processors.statistics.flink.config;
 
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
 import org.apache.streampipes.svcdiscovery.api.SpConfig;
 import org.apache.streampipes.container.model.PeConfig;
 
@@ -32,7 +33,7 @@ public enum StatisticsFlinkConfig implements PeConfig {
   private final static String service_container_name = "processors-statistics-flink";
 
   StatisticsFlinkConfig() {
-    config = SpConfig.getSpConfig(service_id);
+    config = SpServiceDiscovery.getSpConfig(service_id);
 
     config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
     config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/apache/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java b/streampipes-processors-transformation-flink/src/main/java/org/apache/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java
index 6872ec8..0f06da5 100644
--- a/streampipes-processors-transformation-flink/src/main/java/org/apache/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java
+++ b/streampipes-processors-transformation-flink/src/main/java/org/apache/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java
@@ -18,6 +18,7 @@
 
 package org.apache.streampipes.processors.transformation.flink.config;
 
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
 import org.apache.streampipes.svcdiscovery.api.SpConfig;
 import org.apache.streampipes.container.model.PeConfig;
 
@@ -32,7 +33,7 @@ public enum TransformationFlinkConfig implements PeConfig {
   private final static String service_container_name = "processors-transformation-flink";
 
   TransformationFlinkConfig() {
-    config = SpConfig.getSpConfig(service_id);
+    config = SpServiceDiscovery.getSpConfig(service_id);
 
     config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
     config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-sinks-databases-flink/src/main/java/org/apache/streampipes/sinks/databases/flink/config/DatabasesFlinkConfig.java b/streampipes-sinks-databases-flink/src/main/java/org/apache/streampipes/sinks/databases/flink/config/DatabasesFlinkConfig.java
index ee94215..739e27d 100644
--- a/streampipes-sinks-databases-flink/src/main/java/org/apache/streampipes/sinks/databases/flink/config/DatabasesFlinkConfig.java
+++ b/streampipes-sinks-databases-flink/src/main/java/org/apache/streampipes/sinks/databases/flink/config/DatabasesFlinkConfig.java
@@ -18,6 +18,7 @@
 
 package org.apache.streampipes.sinks.databases.flink.config;
 
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
 import org.apache.streampipes.svcdiscovery.api.SpConfig;
 import org.apache.streampipes.container.model.PeConfig;
 
@@ -32,7 +33,7 @@ public enum DatabasesFlinkConfig implements PeConfig {
   private final static String service_container_name = "sinks-databases-flink";
 
   DatabasesFlinkConfig() {
-    config = SpConfig.getSpConfig(service_id);
+    config = SpServiceDiscovery.getSpConfig(service_id);
 
     config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
     config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");