You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2021/07/21 10:43:09 UTC
[incubator-streampipes-extensions] branch STREAMPIPES-319 updated:
[STREAMPIPES-319] Refactor flink config
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch STREAMPIPES-319
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes-extensions.git
The following commit(s) were added to refs/heads/STREAMPIPES-319 by this push:
new f4f27c3 [STREAMPIPES-319] Refactor flink config
f4f27c3 is described below
commit f4f27c39d888ae9e165c9b24ef79d63603ab2a2b
Author: Dominik Riemer <ri...@fzi.de>
AuthorDate: Wed Jul 21 12:42:54 2021 +0200
[STREAMPIPES-319] Refactor flink config
---
.../aggregation/flink/config/AggregationFlinkConfig.java | 3 ++-
.../flink/processor/aggregation/AggregationController.java | 6 +++++-
.../aggregation/flink/processor/count/CountController.java | 6 +++++-
.../flink/processor/eventcount/EventCountController.java | 7 ++++++-
.../aggregation/flink/processor/rate/EventRateController.java | 8 ++++++--
.../processors/enricher/flink/config/EnricherFlinkConfig.java | 4 +++-
.../detection/flink/config/PatternDetectionFlinkConfig.java | 3 ++-
.../processors/statistics/flink/config/StatisticsFlinkConfig.java | 3 ++-
.../transformation/flink/config/TransformationFlinkConfig.java | 3 ++-
.../sinks/databases/flink/config/DatabasesFlinkConfig.java | 3 ++-
10 files changed, 35 insertions(+), 11 deletions(-)
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java
index 134a6cf..c925418 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.aggregation.flink.config;
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.SpConfig;
import org.apache.streampipes.container.model.PeConfig;
@@ -33,7 +34,7 @@ public enum AggregationFlinkConfig implements PeConfig {
AggregationFlinkConfig() {
- config = SpConfig.getSpConfig(service_id);
+ config = SpServiceDiscovery.getSpConfig(service_id);
config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java
index 9d18106..d8baeca 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java
@@ -19,8 +19,10 @@
package org.apache.streampipes.processors.aggregation.flink.processor.aggregation;
import org.apache.commons.lang.StringUtils;
+import org.apache.streampipes.client.StreamPipesClient;
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.container.api.ResolvesContainerProvidedOutputStrategy;
+import org.apache.streampipes.container.config.ConfigExtractor;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -95,7 +97,9 @@ public class AggregationController extends FlinkDataProcessorDeclarer<Aggregatio
@Override
public FlinkDataProcessorRuntime<AggregationParameters> getRuntime(DataProcessorInvocation graph,
- ProcessingElementParameterExtractor extractor) {
+ ProcessingElementParameterExtractor extractor,
+ ConfigExtractor configExtractor,
+ StreamPipesClient streamPipesClient) {
List<String> groupBy = extractor.mappingPropertyValues("groupBy");
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/count/CountController.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/count/CountController.java
index d635aed..1e757b1 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/count/CountController.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/count/CountController.java
@@ -18,6 +18,8 @@
package org.apache.streampipes.processors.aggregation.flink.processor.count;
+import org.apache.streampipes.client.StreamPipesClient;
+import org.apache.streampipes.container.config.ConfigExtractor;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -72,7 +74,9 @@ public class CountController extends FlinkDataProcessorDeclarer<CountParameters>
@Override
public FlinkDataProcessorRuntime<CountParameters> getRuntime(DataProcessorInvocation graph,
- ProcessingElementParameterExtractor extractor) {
+ ProcessingElementParameterExtractor extractor,
+ ConfigExtractor configExtractor,
+ StreamPipesClient streamPipesClient) {
Integer timeWindowSize = extractor.singleValueParameter(TIME_WINDOW_KEY, Integer.class);
String scale = extractor.selectedSingleValueInternalName(SCALE_KEY, String.class);
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/eventcount/EventCountController.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/eventcount/EventCountController.java
index 4700330..b5f3e60 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/eventcount/EventCountController.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/eventcount/EventCountController.java
@@ -17,6 +17,8 @@
*/
package org.apache.streampipes.processors.aggregation.flink.processor.eventcount;
+import org.apache.streampipes.client.StreamPipesClient;
+import org.apache.streampipes.container.config.ConfigExtractor;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -67,7 +69,10 @@ public class EventCountController extends FlinkDataProcessorDeclarer<EventCountP
}
@Override
- public FlinkDataProcessorRuntime<EventCountParameters> getRuntime(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ public FlinkDataProcessorRuntime<EventCountParameters> getRuntime(DataProcessorInvocation graph,
+ ProcessingElementParameterExtractor extractor,
+ ConfigExtractor configExtractor,
+ StreamPipesClient streamPipesClient) {
Integer timeWindowSize = extractor.singleValueParameter(TIME_WINDOW_KEY, Integer.class);
String scale = extractor.selectedSingleValueInternalName(SCALE_KEY, String.class);
EventCountParameters staticParam = new EventCountParameters(graph, timeWindowSize, scale);
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java
index 7522289..d58d76c 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/apache/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java
@@ -18,6 +18,8 @@
package org.apache.streampipes.processors.aggregation.flink.processor.rate;
+import org.apache.streampipes.client.StreamPipesClient;
+import org.apache.streampipes.container.config.ConfigExtractor;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -57,7 +59,9 @@ public class EventRateController extends FlinkDataProcessorDeclarer<EventRatePar
@Override
public FlinkDataProcessorRuntime<EventRateParameter> getRuntime(DataProcessorInvocation graph,
- ProcessingElementParameterExtractor extractor) {
+ ProcessingElementParameterExtractor extractor,
+ ConfigExtractor configExtractor,
+ StreamPipesClient streamPipesClient) {
Integer avgRate = extractor.singleValueParameter(RATE_KEY, Integer.class);
EventRateParameter staticParam = new EventRateParameter(graph, avgRate);
@@ -65,4 +69,4 @@ public class EventRateController extends FlinkDataProcessorDeclarer<EventRatePar
return new EventRateProgram(staticParam, AggregationFlinkConfig.INSTANCE.getDebug());
}
-}
\ No newline at end of file
+}
diff --git a/streampipes-processors-enricher-flink/src/main/java/org/apache/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java b/streampipes-processors-enricher-flink/src/main/java/org/apache/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java
index 909c187..a1b3bd5 100644
--- a/streampipes-processors-enricher-flink/src/main/java/org/apache/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java
+++ b/streampipes-processors-enricher-flink/src/main/java/org/apache/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.enricher.flink.config;
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.SpConfig;
import org.apache.streampipes.container.model.PeConfig;
@@ -30,8 +31,9 @@ public enum EnricherFlinkConfig implements PeConfig {
private final static String service_id = "pe/org.apache.streampipes.processors.enricher.flink";
private final static String service_name = "Processors Enricher Flink";
private final static String service_container_name = "processors-enricher-flink";
+
EnricherFlinkConfig() {
- config = SpConfig.getSpConfig(service_id);
+ config = SpServiceDiscovery.getSpConfig(service_id);
config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/apache/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/apache/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java
index fd12d75..47ffb22 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/apache/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/apache/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.pattern.detection.flink.config;
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.SpConfig;
import org.apache.streampipes.container.model.PeConfig;
@@ -32,7 +33,7 @@ public enum PatternDetectionFlinkConfig implements PeConfig {
private final static String service_container_name = "processors-pattern-detection-flink";
PatternDetectionFlinkConfig() {
- config = SpConfig.getSpConfig(service_id);
+ config = SpServiceDiscovery.getSpConfig(service_id);
config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/apache/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java b/streampipes-processors-statistics-flink/src/main/java/org/apache/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java
index da31662..d240b4e 100644
--- a/streampipes-processors-statistics-flink/src/main/java/org/apache/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java
+++ b/streampipes-processors-statistics-flink/src/main/java/org/apache/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.statistics.flink.config;
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.SpConfig;
import org.apache.streampipes.container.model.PeConfig;
@@ -32,7 +33,7 @@ public enum StatisticsFlinkConfig implements PeConfig {
private final static String service_container_name = "processors-statistics-flink";
StatisticsFlinkConfig() {
- config = SpConfig.getSpConfig(service_id);
+ config = SpServiceDiscovery.getSpConfig(service_id);
config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/apache/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java b/streampipes-processors-transformation-flink/src/main/java/org/apache/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java
index 6872ec8..0f06da5 100644
--- a/streampipes-processors-transformation-flink/src/main/java/org/apache/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java
+++ b/streampipes-processors-transformation-flink/src/main/java/org/apache/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.transformation.flink.config;
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.SpConfig;
import org.apache.streampipes.container.model.PeConfig;
@@ -32,7 +33,7 @@ public enum TransformationFlinkConfig implements PeConfig {
private final static String service_container_name = "processors-transformation-flink";
TransformationFlinkConfig() {
- config = SpConfig.getSpConfig(service_id);
+ config = SpServiceDiscovery.getSpConfig(service_id);
config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
diff --git a/streampipes-sinks-databases-flink/src/main/java/org/apache/streampipes/sinks/databases/flink/config/DatabasesFlinkConfig.java b/streampipes-sinks-databases-flink/src/main/java/org/apache/streampipes/sinks/databases/flink/config/DatabasesFlinkConfig.java
index ee94215..739e27d 100644
--- a/streampipes-sinks-databases-flink/src/main/java/org/apache/streampipes/sinks/databases/flink/config/DatabasesFlinkConfig.java
+++ b/streampipes-sinks-databases-flink/src/main/java/org/apache/streampipes/sinks/databases/flink/config/DatabasesFlinkConfig.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.sinks.databases.flink.config;
+import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.SpConfig;
import org.apache.streampipes.container.model.PeConfig;
@@ -32,7 +33,7 @@ public enum DatabasesFlinkConfig implements PeConfig {
private final static String service_container_name = "sinks-databases-flink";
DatabasesFlinkConfig() {
- config = SpConfig.getSpConfig(service_id);
+ config = SpServiceDiscovery.getSpConfig(service_id);
config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");