You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2021/01/24 21:52:29 UTC
[incubator-streampipes-extensions] branch dev updated:
[STREAMPIPES-288] Add Siddhi-based count aggregation
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes-extensions.git
The following commit(s) were added to refs/heads/dev by this push:
new 57d5250 [STREAMPIPES-288] Add Siddhi-based count aggregation
57d5250 is described below
commit 57d52503fe53d6c1cea538f4937caf6f80a6e4e5
Author: Dominik Riemer <ri...@fzi.de>
AuthorDate: Sun Jan 24 22:52:16 2021 +0100
[STREAMPIPES-288] Add Siddhi-based count aggregation
---
.../processors/siddhi/FiltersSiddhiInit.java | 4 +-
.../processors/siddhi/count/CountAggregation.java | 96 ++++++++++++++++++++++
.../documentation.md | 61 ++++++++++++++
.../strings.en | 11 +++
4 files changed, 171 insertions(+), 1 deletion(-)
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/FiltersSiddhiInit.java b/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/FiltersSiddhiInit.java
index 565f415..b04cbce 100644
--- a/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/FiltersSiddhiInit.java
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/FiltersSiddhiInit.java
@@ -28,6 +28,7 @@ import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory;
import org.apache.streampipes.messaging.kafka.SpKafkaProtocolFactory;
import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory;
import org.apache.streampipes.processors.siddhi.config.FilterSiddhiConfig;
+import org.apache.streampipes.processors.siddhi.count.CountAggregation;
import org.apache.streampipes.processors.siddhi.filter.NumericalFilterController;
import org.apache.streampipes.processors.siddhi.listcollector.ListCollector;
import org.apache.streampipes.processors.siddhi.listfilter.ListFilter;
@@ -40,7 +41,8 @@ public class FiltersSiddhiInit extends StandaloneModelSubmitter {
.add(new TrendController())
.add(new NumericalFilterController())
.add(new ListFilter())
- .add(new ListCollector());
+ .add(new ListCollector())
+ .add(new CountAggregation());
// TODO: currently not working
// .add(new StreamStopController())
// .add(new FrequencyController())
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/count/CountAggregation.java b/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/count/CountAggregation.java
new file mode 100644
index 0000000..273dda0
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/count/CountAggregation.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.processors.siddhi.count;
+
+import org.apache.streampipes.model.DataProcessorType;
+import org.apache.streampipes.model.graph.DataProcessorDescription;
+import org.apache.streampipes.model.schema.PropertyScope;
+import org.apache.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.apache.streampipes.sdk.helpers.*;
+import org.apache.streampipes.sdk.utils.Assets;
+import org.apache.streampipes.wrapper.siddhi.engine.StreamPipesSiddhiProcessor;
+import org.apache.streampipes.wrapper.siddhi.model.SiddhiProcessorParams;
+import org.apache.streampipes.wrapper.siddhi.query.FromClause;
+import org.apache.streampipes.wrapper.siddhi.query.GroupByClause;
+import org.apache.streampipes.wrapper.siddhi.query.SelectClause;
+import org.apache.streampipes.wrapper.siddhi.query.expression.Expressions;
+import org.apache.streampipes.wrapper.siddhi.query.expression.SiddhiTimeUnit;
+import org.apache.streampipes.wrapper.standalone.ProcessorParams;
+
+public class CountAggregation extends StreamPipesSiddhiProcessor {
+
+ private static final String TIME_WINDOW_KEY = "time-window";
+ private static final String SCALE_KEY = "scale";
+ private static final String COUNT_MAPPING = "count-mapping";
+
+ static final String HOURS_INTERNAL_NAME = "HOURS";
+ static final String MINUTES_INTERNAL_NAME = "MINUTES";
+ static final String SECONDS_INTERNAL_NAME = "SECONDS";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.apache.streampipes.processors.siddhi.count")
+ .category(DataProcessorType.AGGREGATE)
+ .withAssets(Assets.DOCUMENTATION, Assets.ICON)
+ .withLocales(Locales.EN)
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.anyProperty(),
+ Labels.withId(COUNT_MAPPING), PropertyScope.DIMENSION_PROPERTY)
+ .build())
+ .outputStrategy(OutputStrategies.fixed(EpProperties.stringEp(Labels.empty(), "value",
+ "http://schema.org/Text"), EpProperties.integerEp(Labels.empty(), "count",
+ "http://schema.org/Number")))
+ .requiredIntegerParameter(Labels.withId(TIME_WINDOW_KEY))
+ .requiredSingleValueSelection(Labels.withId(SCALE_KEY),
+ Options.from(new Tuple2<>("Hours", HOURS_INTERNAL_NAME),
+ new Tuple2<>("Minutes", MINUTES_INTERNAL_NAME),
+ new Tuple2<>("Seconds", SECONDS_INTERNAL_NAME)))
+ .build();
+ }
+
+ @Override
+ public String fromStatement(SiddhiProcessorParams<ProcessorParams> siddhiParams) {
+ Integer timeWindowSize = siddhiParams.getParams().extractor().singleValueParameter(TIME_WINDOW_KEY, Integer.class);
+ String scale = siddhiParams.getParams().extractor().selectedSingleValueInternalName(SCALE_KEY, String.class);
+
+ FromClause fromClause = FromClause.create();
+ fromClause.add(Expressions.stream(siddhiParams.getInputStreamNames().get(0), Expressions.timeWindow(timeWindowSize, toTimeUnit(scale))));
+
+ return fromClause.toSiddhiEpl();
+ }
+
+ private SiddhiTimeUnit toTimeUnit(String scale) {
+ return SiddhiTimeUnit.valueOf(scale);
+ }
+
+ @Override
+ public String selectStatement(SiddhiProcessorParams<ProcessorParams> siddhiParams) {
+ String fieldToCount = siddhiParams.getParams().extractor().mappingPropertyValue(COUNT_MAPPING);
+ return SelectClause.create(Expressions.as(Expressions.property(fieldToCount), "value"),
+ Expressions.as(Expressions.count(Expressions.property(fieldToCount)), "count"))
+ .toSiddhiEpl();
+ }
+
+ @Override
+ public String groupByStatement(SiddhiProcessorParams<ProcessorParams> siddhiParams) {
+ String fieldToCount = siddhiParams.getParams().extractor().mappingPropertyValue(COUNT_MAPPING);
+ return GroupByClause.create(Expressions.property(fieldToCount)).toSiddhiEpl();
+ }
+}
diff --git a/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/documentation.md b/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/documentation.md
new file mode 100644
index 0000000..fe7d1c8
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/documentation.md
@@ -0,0 +1,61 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+## Count Aggregation
+
+<p align="center">
+ <img src="icon.png" width="150px;" class="pe-image-documentation"/>
+</p>
+
+***
+
+## Description
+
+Performs an aggregation based on a given field and outputs the number of occurrences.
+Example: Count the number of vehicle positions per vehicleId.
+The Count aggregation requires a time window, used to perform the count aggregation and a field used to aggregate
+values.
+
+***
+
+## Required input
+
+
+***
+
+## Configuration
+
+### FieldToCount
+Specifies the field containing the values that should be counted.
+
+### TimeWindowSize
+Specifies the size of the time window and consequently the number of values that are aggregated each time.
+
+### Time Window Scale
+Specifies the scale/unit of the time window. There are three different time scales to choose from: seconds, minutes or hours.
+
+## Output
+The output event is composed of two fields. The field "value" specifies the value to count.
+The second field "count" returns the number of occurrences.
+Example:
+```
+{
+ 'value': 'vehicleId',
+ 'count': 12
+}
+```
diff --git a/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/strings.en b/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/strings.en
new file mode 100644
index 0000000..c03282c
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/strings.en
@@ -0,0 +1,11 @@
+org.apache.streampipes.processors.siddhi.count.title=Count Aggregation
+org.apache.streampipes.processors.siddhi.count.description=Performs an aggregation based on a given event property and outputs the number of occurrences.
+
+count-mapping.title=Field to count
+count-mapping.description=The field that contains the values which should be counted
+
+time-window.title=Time Window Size
+time-window.description=Size of the time window
+
+scale.title=Time Window Scale
+scale.description=