You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2022/11/03 13:41:09 UTC
[incubator-streampipes] branch dev updated: Add new Moving Average Processor
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 6d7c6aa1a Add new Moving Average Processor
new f83036a9d Merge pull request #131 from SvenO3/dev
6d7c6aa1a is described below
commit 6d7c6aa1ad2ab2f53cc7340c90cad6345998fb50
Author: Sven Oehler <oe...@web.de>
AuthorDate: Wed Nov 2 17:18:33 2022 +0100
Add new Moving Average Processor
---
.../processors/filters/jvm/FiltersJvmInit.java | 4 +-
.../movingaverage/MovingAverageProcessor.java | 98 ++++++++++++++++++++++
.../movingaverage/util/MovingAverageFilter.java | 38 +++++++++
.../processor/movingaverage/util/MovingFilter.java | 48 +++++++++++
.../movingaverage/util/MovingMedianFilter.java | 42 ++++++++++
.../documentation.md | 40 +++++++++
.../strings.en | 11 +++
7 files changed, 280 insertions(+), 1 deletion(-)
diff --git a/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/FiltersJvmInit.java b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/FiltersJvmInit.java
index 25af677cc..f6d0aef80 100644
--- a/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/FiltersJvmInit.java
+++ b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/FiltersJvmInit.java
@@ -33,6 +33,7 @@ import org.apache.streampipes.processors.filters.jvm.processor.compose.ComposePr
import org.apache.streampipes.processors.filters.jvm.processor.enrich.MergeByEnrichProcessor;
import org.apache.streampipes.processors.filters.jvm.processor.limit.RateLimitProcessor;
import org.apache.streampipes.processors.filters.jvm.processor.merge.MergeByTimeProcessor;
+import org.apache.streampipes.processors.filters.jvm.processor.movingaverage.MovingAverageProcessor;
import org.apache.streampipes.processors.filters.jvm.processor.numericalfilter.NumericalFilterProcessor;
import org.apache.streampipes.processors.filters.jvm.processor.numericaltextfilter.NumericalTextFilterProcessor;
import org.apache.streampipes.processors.filters.jvm.processor.throughputmon.ThroughputMonitorProcessor;
@@ -65,7 +66,8 @@ public class FiltersJvmInit extends StandaloneModelSubmitter {
new MergeBySchemaProcessor(),
new ComposeProcessor(),
new NumericalTextFilterProcessor(),
- new RateLimitProcessor())
+ new RateLimitProcessor(),
+ new MovingAverageProcessor())
.registerMessagingFormats(
new JsonDataFormatFactory(),
new CborDataFormatFactory(),
diff --git a/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/MovingAverageProcessor.java b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/MovingAverageProcessor.java
new file mode 100644
index 000000000..fb2fb3dea
--- /dev/null
+++ b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/MovingAverageProcessor.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.processors.filters.jvm.processor.movingaverage;
+
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.model.DataProcessorType;
+import org.apache.streampipes.model.graph.DataProcessorDescription;
+import org.apache.streampipes.model.runtime.Event;
+import org.apache.streampipes.model.schema.PropertyScope;
+import org.apache.streampipes.processors.filters.jvm.processor.movingaverage.util.*;
+import org.apache.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.apache.streampipes.sdk.helpers.*;
+import org.apache.streampipes.sdk.utils.Assets;
+import org.apache.streampipes.vocabulary.SO;
+import org.apache.streampipes.wrapper.context.EventProcessorRuntimeContext;
+import org.apache.streampipes.wrapper.routing.SpOutputCollector;
+import org.apache.streampipes.wrapper.standalone.ProcessorParams;
+import org.apache.streampipes.wrapper.standalone.StreamPipesDataProcessor;
+
+;
+
+public class MovingAverageProcessor extends StreamPipesDataProcessor {
+
+ private final String RESULT_FIELD = "filterResult";
+ private final String NUMBER_VALUE = "number";
+ private final String N_VALUE = "n";
+ private static final String METHOD_KEY = "method";
+ private static final String MEAN_INTERNAL_NAME = "MEAN";
+ private static final String MEDIAN_INTERNAL_NAME = "MEDIAN";
+
+
+ private String numberName;
+ private MovingFilter filter;
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.apache.streampipes.processors.filters.jvm.movingaverage")
+ .category(DataProcessorType.FILTER)
+ .withLocales(Locales.EN)
+ .withAssets(Assets.DOCUMENTATION)
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.numberReq(),
+ Labels.withId(NUMBER_VALUE),
+ PropertyScope.NONE)
+ .build())
+ .requiredIntegerParameter(Labels.withId(N_VALUE))
+ .requiredSingleValueSelection(Labels.withId(METHOD_KEY),
+ Options.from(new Tuple2<>("mean", MEAN_INTERNAL_NAME),
+ new Tuple2<>("median", MEDIAN_INTERNAL_NAME)))
+ .outputStrategy(
+ OutputStrategies.append(
+ EpProperties.numberEp(Labels.empty(), RESULT_FIELD, SO.Number)))
+ .build();
+ }
+
+ @Override
+ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException {
+ this.numberName = parameters.extractor().mappingPropertyValue(NUMBER_VALUE);
+ int n = parameters.extractor().singleValueParameter(N_VALUE, Integer.class);
+ String methode = parameters.extractor().selectedSingleValueInternalName(METHOD_KEY, String.class);
+ if (methode.equals(MEDIAN_INTERNAL_NAME)){
+ filter = new MovingMedianFilter(n);
+ }
+ else{
+ filter = new MovingAverageFilter(n);
+ }
+ }
+
+ @Override
+ public void onEvent(Event event, SpOutputCollector collector) throws SpRuntimeException {
+ Double result = filter.update(event.getFieldBySelector(numberName).getAsPrimitive().getAsDouble());
+ event.addField(RESULT_FIELD, result);
+ collector.collect(event);
+ }
+
+ @Override
+ public void onDetach() throws SpRuntimeException {
+
+ }
+}
\ No newline at end of file
diff --git a/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/util/MovingAverageFilter.java b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/util/MovingAverageFilter.java
new file mode 100644
index 000000000..31fce6397
--- /dev/null
+++ b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/util/MovingAverageFilter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.processors.filters.jvm.processor.movingaverage.util;
+
+import java.util.List;
+
+public class MovingAverageFilter extends MovingFilter {
+
+ public MovingAverageFilter(int n) {
+ super(n);
+ }
+
+ @Override
+ protected double filterMeasurement(List<Double> measurements) {
+ double sum = 0;
+ for (double m : measurements) {
+ sum += m;
+ }
+ return sum / measurements.size();
+ }
+
+}
diff --git a/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/util/MovingFilter.java b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/util/MovingFilter.java
new file mode 100644
index 000000000..f989675ff
--- /dev/null
+++ b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/util/MovingFilter.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.processors.filters.jvm.processor.movingaverage.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class MovingFilter {
+ private int n;
+ private double filteredMeasurement;
+ private List<Double> measurements;
+
+ public MovingFilter(int n) {
+ this.n = n;
+ measurements = new ArrayList<Double>();
+ }
+
+ public double update(double newMeasurement) {
+ measurements.add(newMeasurement);
+ if (measurements.size() > n) {
+ measurements.remove(0);
+ }
+ filteredMeasurement = filterMeasurement(measurements);
+ return filteredMeasurement;
+ }
+
+ public double getFilteredMeasurement() {
+ return filteredMeasurement;
+ }
+
+ protected abstract double filterMeasurement(List<Double> measurements);
+}
diff --git a/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/util/MovingMedianFilter.java b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/util/MovingMedianFilter.java
new file mode 100644
index 000000000..d2c6ad3c3
--- /dev/null
+++ b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/java/org/apache/streampipes/processors/filters/jvm/processor/movingaverage/util/MovingMedianFilter.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.processors.filters.jvm.processor.movingaverage.util;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class MovingMedianFilter extends MovingFilter {
+
+ public MovingMedianFilter(int n) {
+ super(n);
+ }
+
+ @Override
+ protected double filterMeasurement(List<Double> measurements) {
+ List<Double> sorted = new ArrayList<Double>();
+ sorted.addAll(measurements);
+ Collections.sort(sorted);
+ if (sorted.size()%2 == 0){
+ return (sorted.get(sorted.size()/2) + sorted.get(sorted.size()/2 -1))/2;
+ }
+ return sorted.get(sorted.size() / 2);
+ }
+
+}
\ No newline at end of file
diff --git a/streampipes-extensions/streampipes-processors-filters-jvm/src/main/resources/org.apache.streampipes.processors.filters.jvm.movingaverage/documentation.md b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/resources/org.apache.streampipes.processors.filters.jvm.movingaverage/documentation.md
new file mode 100644
index 000000000..e9debe873
--- /dev/null
+++ b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/resources/org.apache.streampipes.processors.filters.jvm.movingaverage/documentation.md
@@ -0,0 +1,40 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+## Moving Average
+***
+
+
+## Description
+
+Smooths the data stream by the mean/median of the last n values.
+
+***
+
+## Required input
+A numerical field is required.
+***
+
+## Configuration
+### N Value
+Specifies the number of previous data points which are used to smooth the data.
+### Method
+Specifies the method which is used to smooth the data. Choose between mean and median.
+
+## Output
+Appends a field with the smoothed data.
\ No newline at end of file
diff --git a/streampipes-extensions/streampipes-processors-filters-jvm/src/main/resources/org.apache.streampipes.processors.filters.jvm.movingaverage/strings.en b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/resources/org.apache.streampipes.processors.filters.jvm.movingaverage/strings.en
new file mode 100644
index 000000000..f9f58cc43
--- /dev/null
+++ b/streampipes-extensions/streampipes-processors-filters-jvm/src/main/resources/org.apache.streampipes.processors.filters.jvm.movingaverage/strings.en
@@ -0,0 +1,11 @@
+org.apache.streampipes.processors.filters.jvm.movingaverage.title = Moving Average
+org.apache.streampipes.processors.filters.jvm.movingaverage.description = Smooths the data stream by the mean/median of the last n values
+
+number.title = Numerical Field
+number.description = Select the field to be smoothed
+
+n.title = N Value
+n.description = Number of previous data points which are used to smooth the data
+
+method.title = Select Method
+method.description = Method used to smooth the data
\ No newline at end of file