You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2021/01/24 21:52:29 UTC

[incubator-streampipes-extensions] branch dev updated: [STREAMPIPES-288] Add Siddhi-based count aggregation

This is an automated email from the ASF dual-hosted git repository.

riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes-extensions.git


The following commit(s) were added to refs/heads/dev by this push:
     new 57d5250  [STREAMPIPES-288] Add Siddhi-based count aggregation
57d5250 is described below

commit 57d52503fe53d6c1cea538f4937caf6f80a6e4e5
Author: Dominik Riemer <ri...@fzi.de>
AuthorDate: Sun Jan 24 22:52:16 2021 +0100

    [STREAMPIPES-288] Add Siddhi-based count aggregation
---
 .../processors/siddhi/FiltersSiddhiInit.java       |  4 +-
 .../processors/siddhi/count/CountAggregation.java  | 96 ++++++++++++++++++++++
 .../documentation.md                               | 61 ++++++++++++++
 .../strings.en                                     | 11 +++
 4 files changed, 171 insertions(+), 1 deletion(-)

diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/FiltersSiddhiInit.java b/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/FiltersSiddhiInit.java
index 565f415..b04cbce 100644
--- a/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/FiltersSiddhiInit.java
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/FiltersSiddhiInit.java
@@ -28,6 +28,7 @@ import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory;
 import org.apache.streampipes.messaging.kafka.SpKafkaProtocolFactory;
 import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory;
 import org.apache.streampipes.processors.siddhi.config.FilterSiddhiConfig;
+import org.apache.streampipes.processors.siddhi.count.CountAggregation;
 import org.apache.streampipes.processors.siddhi.filter.NumericalFilterController;
 import org.apache.streampipes.processors.siddhi.listcollector.ListCollector;
 import org.apache.streampipes.processors.siddhi.listfilter.ListFilter;
@@ -40,7 +41,8 @@ public class FiltersSiddhiInit extends StandaloneModelSubmitter {
             .add(new TrendController())
             .add(new NumericalFilterController())
             .add(new ListFilter())
-            .add(new ListCollector());
+            .add(new ListCollector())
+            .add(new CountAggregation());
     // TODO: currently not working
 //            .add(new StreamStopController())
 //            .add(new FrequencyController())
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/count/CountAggregation.java b/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/count/CountAggregation.java
new file mode 100644
index 0000000..273dda0
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/apache/streampipes/processors/siddhi/count/CountAggregation.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.processors.siddhi.count;
+
+import org.apache.streampipes.model.DataProcessorType;
+import org.apache.streampipes.model.graph.DataProcessorDescription;
+import org.apache.streampipes.model.schema.PropertyScope;
+import org.apache.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.apache.streampipes.sdk.helpers.*;
+import org.apache.streampipes.sdk.utils.Assets;
+import org.apache.streampipes.wrapper.siddhi.engine.StreamPipesSiddhiProcessor;
+import org.apache.streampipes.wrapper.siddhi.model.SiddhiProcessorParams;
+import org.apache.streampipes.wrapper.siddhi.query.FromClause;
+import org.apache.streampipes.wrapper.siddhi.query.GroupByClause;
+import org.apache.streampipes.wrapper.siddhi.query.SelectClause;
+import org.apache.streampipes.wrapper.siddhi.query.expression.Expressions;
+import org.apache.streampipes.wrapper.siddhi.query.expression.SiddhiTimeUnit;
+import org.apache.streampipes.wrapper.standalone.ProcessorParams;
+
+public class CountAggregation extends StreamPipesSiddhiProcessor {
+
+  private static final String TIME_WINDOW_KEY = "time-window";
+  private static final String SCALE_KEY = "scale";
+  private static final String COUNT_MAPPING = "count-mapping";
+
+  static final String HOURS_INTERNAL_NAME = "HOURS";
+  static final String MINUTES_INTERNAL_NAME = "MINUTES";
+  static final String SECONDS_INTERNAL_NAME = "SECONDS";
+
+  @Override
+  public DataProcessorDescription declareModel() {
+    return ProcessingElementBuilder.create("org.apache.streampipes.processors.siddhi.count")
+            .category(DataProcessorType.AGGREGATE)
+            .withAssets(Assets.DOCUMENTATION, Assets.ICON)
+            .withLocales(Locales.EN)
+            .requiredStream(StreamRequirementsBuilder
+                    .create()
+                    .requiredPropertyWithUnaryMapping(EpRequirements.anyProperty(),
+                            Labels.withId(COUNT_MAPPING), PropertyScope.DIMENSION_PROPERTY)
+                    .build())
+            .outputStrategy(OutputStrategies.fixed(EpProperties.stringEp(Labels.empty(), "value",
+                    "http://schema.org/Text"), EpProperties.integerEp(Labels.empty(), "count",
+                    "http://schema.org/Number")))
+            .requiredIntegerParameter(Labels.withId(TIME_WINDOW_KEY))
+            .requiredSingleValueSelection(Labels.withId(SCALE_KEY),
+                    Options.from(new Tuple2<>("Hours", HOURS_INTERNAL_NAME),
+                            new Tuple2<>("Minutes", MINUTES_INTERNAL_NAME),
+                            new Tuple2<>("Seconds", SECONDS_INTERNAL_NAME)))
+            .build();
+  }
+
+  @Override
+  public String fromStatement(SiddhiProcessorParams<ProcessorParams> siddhiParams) {
+    Integer timeWindowSize = siddhiParams.getParams().extractor().singleValueParameter(TIME_WINDOW_KEY, Integer.class);
+    String scale = siddhiParams.getParams().extractor().selectedSingleValueInternalName(SCALE_KEY, String.class);
+
+    FromClause fromClause = FromClause.create();
+    fromClause.add(Expressions.stream(siddhiParams.getInputStreamNames().get(0), Expressions.timeWindow(timeWindowSize, toTimeUnit(scale))));
+
+    return fromClause.toSiddhiEpl();
+  }
+
+  private SiddhiTimeUnit toTimeUnit(String scale) {
+    return SiddhiTimeUnit.valueOf(scale);
+  }
+
+  @Override
+  public String selectStatement(SiddhiProcessorParams<ProcessorParams> siddhiParams) {
+    String fieldToCount = siddhiParams.getParams().extractor().mappingPropertyValue(COUNT_MAPPING);
+    return SelectClause.create(Expressions.as(Expressions.property(fieldToCount), "value"),
+            Expressions.as(Expressions.count(Expressions.property(fieldToCount)), "count"))
+            .toSiddhiEpl();
+  }
+
+  @Override
+  public String groupByStatement(SiddhiProcessorParams<ProcessorParams> siddhiParams) {
+    String fieldToCount = siddhiParams.getParams().extractor().mappingPropertyValue(COUNT_MAPPING);
+    return GroupByClause.create(Expressions.property(fieldToCount)).toSiddhiEpl();
+  }
+}
diff --git a/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/documentation.md b/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/documentation.md
new file mode 100644
index 0000000..fe7d1c8
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/documentation.md
@@ -0,0 +1,61 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+## Count Aggregation
+
+<p align="center"> 
+    <img src="icon.png" width="150px;" class="pe-image-documentation"/>
+</p>
+
+***
+
+## Description
+
+Performs an aggregation based on a given field and outputs the number of occurrences.
+Example: Count the number of vehicle positions per vehicleId.
+The Count aggregation requires a time window, used to perform the count aggregation and a field used to aggregate
+values.
+
+***
+
+## Required input
+
+
+***
+
+## Configuration
+
+### FieldToCount    
+Specifies the field containing the values that should be counted.
+
+### TimeWindowSize  
+Specifies the size of the time window and consequently the number of values that are aggregated each time. 
+
+### Time Window Scale
+Specifies the scale/unit of the time window. There are three different time scales to choose from: seconds, minutes or hours.
+
+## Output
+The output event is composed of two fields. The field "value" specifies the value to count.
+The second field "count" returns the number of occurrences.
+Example:
+```
+{
+  'value': 'vehicleId', 
+  'count': 12
+}
+```
diff --git a/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/strings.en b/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/strings.en
new file mode 100644
index 0000000..c03282c
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/resources/org.apache.streampipes.processors.siddhi.count/strings.en
@@ -0,0 +1,11 @@
+org.apache.streampipes.processors.siddhi.count.title=Count Aggregation
+org.apache.streampipes.processors.siddhi.count.description=Performs an aggregation based on a given event property and outputs the number of occurrences.
+
+count-mapping.title=Field to count
+count-mapping.description=The field that contains the values which should be counted
+
+time-window.title=Time Window Size
+time-window.description=Size of the time window
+
+scale.title=Time Window Scale
+scale.description=