You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2025/11/12 12:57:18 UTC
(streampipes) branch 3909-remove-deprecated-flink-wrapper updated: refactor(#3909): Remove flink wrapper
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch 3909-remove-deprecated-flink-wrapper
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/3909-remove-deprecated-flink-wrapper by this push:
new d6c8e3abff refactor(#3909): Remove flink wrapper
d6c8e3abff is described below
commit d6c8e3abff7f36df8139cc353be2222c9ad8d233
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Wed Nov 12 13:57:10 2025 +0100
refactor(#3909): Remove flink wrapper
---
.github/workflows/build.yml | 7 -
LICENSE | 7 -
.../README.md | 38 --
.../pom.xml | 63 ---
.../META-INF/maven/archetype-metadata.xml | 72 ---
.../main/resources/archetype-resources/Dockerfile | 20 -
.../resources/archetype-resources/development/env | 20 -
.../src/main/resources/archetype-resources/pom.xml | 159 -------
.../archetype-resources/src/main/java/Init.java | 64 ---
.../src/main/java/config/ConfigKeys.java | 33 --
.../__packageName__/__classNamePrefix__.java | 42 --
.../__classNamePrefix__Controller.java | 76 ----
.../__classNamePrefix__Parameters.java | 41 --
.../__classNamePrefix__Program.java | 70 ---
.../documentation.md | 42 --
.../icon.png | Bin 13563 -> 0 bytes
.../strings.en | 5 -
.../streampipes-archetype-pe-sinks-flink/README.md | 38 --
.../streampipes-archetype-pe-sinks-flink/pom.xml | 62 ---
.../META-INF/maven/archetype-metadata.xml | 72 ---
.../main/resources/archetype-resources/Dockerfile | 20 -
.../resources/archetype-resources/development/env | 20 -
.../src/main/resources/archetype-resources/pom.xml | 160 -------
.../archetype-resources/src/main/java/Init.java | 63 ---
.../src/main/java/config/ConfigKeys.java | 32 --
.../__classNamePrefix__Controller.java | 74 ---
.../__classNamePrefix__Parameters.java | 52 ---
.../__classNamePrefix__Program.java | 68 ---
.../documentation.md | 39 --
.../__package__.pe.sink.__packageName__/icon.png | Bin 13563 -> 0 bytes
.../__package__.pe.sink.__packageName__/strings.en | 11 -
.../standalone/jobmanager/docker-compose.dev.yml | 20 -
.../standalone/jobmanager/docker-compose.yml | 32 --
.../docker-compose.dev.yml | 21 -
.../pipeline-elements-all-flink/docker-compose.yml | 31 --
.../standalone/taskmanager/docker-compose.dev.yml | 19 -
.../standalone/taskmanager/docker-compose.yml | 37 --
installer/cli/environments/full-monitoring | 1 -
pom.xml | 24 -
.../apache/streampipes/commons/constants/Envs.java | 7 -
.../commons/environment/DefaultEnvironment.java | 15 -
.../commons/environment/Environment.java | 7 -
.../Dockerfile | 20 -
.../development/env | 20 -
.../pom.xml | 122 -----
.../boilerpipe/BoilerpipeDocumentSource.java | 27 --
.../boilerpipe/BoilerpipeExtractor.java | 64 ---
.../kohlschutter/boilerpipe/BoilerpipeFilter.java | 36 --
.../kohlschutter/boilerpipe/BoilerpipeInput.java | 33 --
.../boilerpipe/BoilerpipeProcessingException.java | 41 --
.../boilerpipe/conditions/TextBlockCondition.java | 36 --
.../kohlschutter/boilerpipe/document/Image.java | 108 -----
.../boilerpipe/document/TextBlock.java | 283 ------------
.../boilerpipe/document/TextDocument.java | 133 ------
.../document/TextDocumentStatistics.java | 62 ---
.../boilerpipe/document/package-info.java | 22 -
.../boilerpipe/estimators/SimpleEstimator.java | 61 ---
.../boilerpipe/extractors/ArticleExtractor.java | 64 ---
.../extractors/ArticleSentencesExtractor.java | 45 --
.../boilerpipe/extractors/CanolaExtractor.java | 94 ----
.../boilerpipe/extractors/CommonExtractors.java | 56 ---
.../boilerpipe/extractors/DefaultExtractor.java | 47 --
.../boilerpipe/extractors/ExtractorBase.java | 109 -----
.../extractors/KeepEverythingExtractor.java | 39 --
.../KeepEverythingWithMinKWordsExtractor.java | 44 --
.../extractors/LargestContentExtractor.java | 50 --
.../extractors/NumWordsRulesExtractor.java | 43 --
.../boilerpipe/extractors/package-info.java | 22 -
.../boilerpipe/filters/debug/PrintDebugFilter.java | 66 ---
.../filters/english/DensityRulesClassifier.java | 110 -----
.../filters/english/HeuristicFilterBase.java | 38 --
.../english/IgnoreBlocksAfterContentFilter.java | 78 ----
.../IgnoreBlocksAfterContentFromEndFilter.java | 74 ---
.../english/KeepLargestFulltextBlockFilter.java | 81 ----
.../filters/english/MinFulltextWordsFilter.java | 59 ---
.../filters/english/NumWordsRulesClassifier.java | 110 -----
.../filters/english/TerminatingBlocksFinder.java | 109 -----
.../boilerpipe/filters/english/package-info.java | 25 -
.../heuristics/AddPrecedingLabelsFilter.java | 82 ----
.../filters/heuristics/ArticleMetadataFilter.java | 63 ---
.../filters/heuristics/BlockProximityFusion.java | 122 -----
.../filters/heuristics/ContentFusion.java | 72 ---
.../heuristics/DocumentTitleMatchClassifier.java | 170 -------
.../heuristics/ExpandTitleToContentFilter.java | 70 ---
.../filters/heuristics/KeepLargestBlockFilter.java | 117 -----
.../boilerpipe/filters/heuristics/LabelFusion.java | 87 ----
.../LargeBlockSameTagLevelToContentFilter.java | 70 ---
.../filters/heuristics/ListAtEndFilter.java | 57 ---
.../heuristics/SimpleBlockFusionProcessor.java | 67 ---
.../TrailingHeadlineToBoilerplateFilter.java | 64 ---
.../filters/heuristics/package-info.java | 22 -
.../filters/simple/BoilerplateBlockFilter.java | 64 ---
.../boilerpipe/filters/simple/InvertedFilter.java | 49 --
.../filters/simple/LabelToBoilerplateFilter.java | 58 ---
.../filters/simple/LabelToContentFilter.java | 54 ---
.../simple/MarkEverythingBoilerplateFilter.java | 49 --
.../simple/MarkEverythingContentFilter.java | 48 --
.../filters/simple/MinClauseWordsFilter.java | 108 -----
.../boilerpipe/filters/simple/MinWordsFilter.java | 53 ---
.../filters/simple/SplitParagraphBlocksFilter.java | 79 ----
.../filters/simple/SurroundingToContentFilter.java | 77 ----
.../boilerpipe/filters/simple/package-info.java | 23 -
.../boilerpipe/labels/ConditionalLabelAction.java | 40 --
.../boilerpipe/labels/DefaultLabels.java | 46 --
.../boilerpipe/labels/LabelAction.java | 47 --
.../com/kohlschutter/boilerpipe/package-info.java | 22 -
.../sax/BoilerpipeHTMLContentHandler.java | 440 ------------------
.../boilerpipe/sax/BoilerpipeHTMLParser.java | 74 ---
.../boilerpipe/sax/BoilerpipeSAXInput.java | 71 ---
.../boilerpipe/sax/CommonTagActions.java | 343 --------------
.../boilerpipe/sax/DefaultTagActionMap.java | 81 ----
.../kohlschutter/boilerpipe/sax/HTMLDocument.java | 56 ---
.../kohlschutter/boilerpipe/sax/HTMLFetcher.java | 91 ----
.../boilerpipe/sax/HTMLHighlighter.java | 504 ---------------------
.../boilerpipe/sax/ImageExtractor.java | 278 ------------
.../boilerpipe/sax/InputSourceable.java | 28 --
.../boilerpipe/sax/MarkupTagAction.java | 117 -----
.../com/kohlschutter/boilerpipe/sax/TagAction.java | 38 --
.../kohlschutter/boilerpipe/sax/TagActionMap.java | 59 ---
.../kohlschutter/boilerpipe/sax/package-info.java | 23 -
.../boilerpipe/util/UnicodeTokenizer.java | 40 --
.../kohlschutter/boilerpipe/util/package-info.java | 23 -
.../flink/AbstractFlinkTransformationProgram.java | 29 --
.../pe/flink/AbstractPatternDetectionProgram.java | 39 --
.../pe/flink/AllFlinkPipelineElementsInit.java | 91 ----
.../streampipes/pe/flink/config/ConfigKeys.java | 28 --
.../pe/flink/extensions/MapKeySelector.java | 43 --
.../pe/flink/extensions/SlidingBatchWindow.java | 50 --
.../flink/extensions/SlidingEventTimeWindow.java | 79 ----
.../flink/extensions/TimestampMappingFunction.java | 28 --
.../pe/flink/processor/absence/Absence.java | 21 -
.../flink/processor/absence/AbsenceController.java | 82 ----
.../flink/processor/absence/AbsenceParameters.java | 55 ---
.../pe/flink/processor/absence/AbsenceProgram.java | 127 ------
.../flink/processor/aggregation/Aggregation.java | 87 ----
.../aggregation/AggregationController.java | 162 -------
.../aggregation/AggregationParameters.java | 83 ----
.../processor/aggregation/AggregationProgram.java | 87 ----
.../processor/aggregation/AggregationType.java | 26 --
.../processor/aggregation/CountAggregation.java | 55 ---
.../processor/aggregation/TimeAggregation.java | 54 ---
.../processor/aggregation/config/aggregation.md | 23 -
.../streampipes/pe/flink/processor/and/And.java | 21 -
.../pe/flink/processor/and/AndController.java | 84 ----
.../pe/flink/processor/and/AndParameters.java | 59 ---
.../pe/flink/processor/and/AndProgram.java | 75 ---
.../pe/flink/processor/and/TimeUnit.java | 22 -
.../pe/flink/processor/and/TimeUnitConverter.java | 34 --
.../boilerplate/BoilerplateController.java | 105 -----
.../boilerplate/BoilerplateParameters.java | 49 --
.../processor/boilerplate/BoilerplateProgram.java | 38 --
.../processor/boilerplate/BoilerplateRemover.java | 104 -----
.../flink/processor/boilerplate/ExtractorMode.java | 28 --
.../pe/flink/processor/boilerplate/OutputMode.java | 25 -
.../flink/processor/common/TimestampExtractor.java | 36 --
.../flink/processor/converter/FieldConverter.java | 62 ---
.../converter/FieldConverterController.java | 155 -------
.../converter/FieldConverterParameters.java | 42 --
.../processor/converter/FieldConverterProgram.java | 35 --
.../pe/flink/processor/count/CountController.java | 86 ----
.../pe/flink/processor/count/CountMapper.java | 38 --
.../pe/flink/processor/count/CountParameters.java | 50 --
.../pe/flink/processor/count/CountProgram.java | 51 ---
.../pe/flink/processor/count/CountTrigger.java | 46 --
.../flink/processor/count/TimeWindowConverter.java | 35 --
.../pe/flink/processor/count/Tuple2MapMapper.java | 39 --
.../processor/eventcount/EventCountController.java | 77 ----
.../processor/eventcount/EventCountMapper.java | 29 --
.../eventcount/EventCountOutputMapper.java | 39 --
.../processor/eventcount/EventCountParameters.java | 42 --
.../processor/eventcount/EventCountProgram.java | 41 --
.../flink/processor/gridenricher/CellOption.java | 128 ------
.../processor/gridenricher/EnrichmentSettings.java | 84 ----
.../gridenricher/SpatialGridCalculator.java | 64 ---
.../gridenricher/SpatialGridConstants.java | 31 --
.../gridenricher/SpatialGridEnricher.java | 61 ---
.../SpatialGridEnrichmentController.java | 102 -----
.../SpatialGridEnrichmentParameters.java | 36 --
.../gridenricher/SpatialGridEnrichmentProgram.java | 36 --
.../MeasurementUnitConverter.java | 58 ---
.../MeasurementUnitConverterController.java | 128 ------
.../MeasurementUnitConverterParameters.java | 52 ---
.../MeasurementUnitConverterProgram.java | 40 --
.../processor/peak/PeakDetectionCalculator.java | 135 ------
.../processor/peak/PeakDetectionController.java | 90 ----
.../processor/peak/PeakDetectionParameters.java | 84 ----
.../flink/processor/peak/PeakDetectionProgram.java | 73 ---
.../processor/peak/utils/SlidingBatchWindow.java | 53 ---
.../pe/flink/processor/rate/EventRate.java | 40 --
.../flink/processor/rate/EventRateController.java | 67 ---
.../flink/processor/rate/EventRateParameter.java | 37 --
.../pe/flink/processor/rate/EventRateProgram.java | 49 --
.../pe/flink/processor/rename/FieldRenamer.java | 48 --
.../processor/rename/FieldRenamerController.java | 68 ---
.../processor/rename/FieldRenamerParameters.java | 52 ---
.../processor/rename/FieldRenamerProgram.java | 38 --
.../pe/flink/processor/sequence/EventStorage.java | 41 --
.../pe/flink/processor/sequence/Sequence.java | 86 ----
.../processor/sequence/SequenceController.java | 67 ---
.../processor/sequence/SequenceParameters.java | 42 --
.../flink/processor/sequence/SequenceProgram.java | 46 --
.../stat/summary/StatisticsSummaryCalculator.java | 64 ---
.../stat/summary/StatisticsSummaryController.java | 107 -----
.../stat/summary/StatisticsSummaryParameters.java | 38 --
.../stat/summary/StatisticsSummaryProgram.java | 36 --
.../window/StatisticsSummaryCalculatorWindow.java | 69 ---
.../window/StatisticsSummaryControllerWindow.java | 117 -----
.../window/StatisticsSummaryParametersWindow.java | 69 ---
.../StatisticsSummaryParamsSerializable.java | 62 ---
.../window/StatisticsSummaryProgramWindow.java | 67 ---
.../processor/timestamp/TimestampController.java | 66 ---
.../processor/timestamp/TimestampEnricher.java | 42 --
.../processor/timestamp/TimestampParameters.java | 38 --
.../processor/timestamp/TimestampProgram.java | 37 --
.../urldereferencing/UrlDereferencing.java | 57 ---
.../UrlDereferencingController.java | 71 ---
.../UrlDereferencingParameter.java | 42 --
.../urldereferencing/UrlDereferencingProgram.java | 38 --
.../pe/flink/processor/wordcount/Word.java | 54 ---
.../processor/wordcount/WordCountController.java | 76 ----
.../processor/wordcount/WordCountParameters.java | 42 --
.../processor/wordcount/WordCountProgram.java | 45 --
.../pe/flink/processor/wordcount/WordSplitter.java | 44 --
.../processor/wordcount/WordToEventConverter.java | 34 --
.../elasticsearch/ElasticSearchController.java | 72 ---
.../elasticsearch/ElasticSearchParameters.java | 58 ---
.../sink/elasticsearch/ElasticSearchProgram.java | 81 ----
.../ElasticsearchIndexRequestBuilder.java | 55 ---
.../elastic/ActionRequestFailureHandler.java | 42 --
.../elastic/BulkProcessorIndexer.java | 55 ---
.../elastic/Elasticsearch6ApiCallBridge.java | 116 -----
.../elastic/ElasticsearchApiCallBridge.java | 97 ----
.../elasticsearch/elastic/ElasticsearchSink.java | 93 ----
.../elastic/ElasticsearchSinkBase.java | 456 -------------------
.../elastic/ElasticsearchSinkFunction.java | 40 --
.../elasticsearch/elastic/NoOpFailureHandler.java | 39 --
.../sink/elasticsearch/elastic/RequestIndexer.java | 37 --
.../documentation.md | 45 --
.../icon.png | Bin 9425 -> 0 bytes
.../strings.en | 32 --
.../documentation.md | 55 ---
.../icon.png | Bin 1854 -> 0 bytes
.../strings.en | 53 ---
.../documentation.md | 60 ---
.../icon.png | Bin 4738 -> 0 bytes
.../strings.en | 29 --
.../documentation.md | 50 --
.../strings.en | 26 --
.../documentation.md | 48 --
.../icon.png | Bin 6197 -> 0 bytes
.../strings.en | 23 -
.../documentation.md | 45 --
.../icon.png | Bin 7429 -> 0 bytes
.../strings.en | 23 -
.../documentation.md | 42 --
.../icon.png | Bin 19386 -> 0 bytes
.../strings.en | 20 -
.../documentation.md | 47 --
.../strings.en | 25 -
.../documentation.md | 47 --
.../icon.png | Bin 2516 -> 0 bytes
.../strings.en | 31 --
.../documentation.md | 47 --
.../icon.png | Bin 20685 -> 0 bytes
.../strings.en | 40 --
.../documentation.md | 47 --
.../icon.png | Bin 710 -> 0 bytes
.../strings.en | 25 -
.../documentation.md | 49 --
.../icon.png | Bin 23118 -> 0 bytes
.../strings.en | 35 --
.../documentation.md | 37 --
.../icon.png | Bin 4378 -> 0 bytes
.../strings.en | 22 -
.../documentation.md | 47 --
.../icon.png | Bin 15515 -> 0 bytes
.../strings.en | 27 --
.../documentation.md | 48 --
.../icon.png | Bin 11482 -> 0 bytes
.../strings.en | 25 -
.../documentation.md | 63 ---
.../icon.png | Bin 11398 -> 0 bytes
.../strings.en | 25 -
.../documentation.md | 62 ---
.../icon.png | Bin 14453 -> 0 bytes
.../strings.en | 25 -
.../documentation.md | 48 --
.../icon.png | Bin 19799 -> 0 bytes
.../strings.en | 25 -
.../documentation.md | 47 --
.../icon.png | Bin 10904 -> 0 bytes
.../strings.en | 25 -
.../documentation.md | 43 --
.../icon.png | Bin 12949 -> 0 bytes
.../strings.en | 28 --
.../documentation.md | 54 ---
.../icon.png | Bin 20411 -> 0 bytes
.../strings.en | 25 -
.../src/main/resources/docker-compose.yml | 25 -
streampipes-wrapper-flink/pom.xml | 160 -------
.../flink/FlinkDataProcessorCompatProgram.java | 76 ----
.../wrapper/flink/FlinkDataProcessorDeclarer.java | 66 ---
.../wrapper/flink/FlinkDataProcessorProgram.java | 30 --
.../wrapper/flink/FlinkDataProcessorRuntime.java | 116 -----
.../wrapper/flink/FlinkDataSinkCompatProgram.java | 38 --
.../wrapper/flink/FlinkDataSinkDeclarer.java | 62 ---
.../wrapper/flink/FlinkDataSinkProgram.java | 30 --
.../wrapper/flink/FlinkDataSinkRuntime.java | 74 ---
.../wrapper/flink/FlinkDeploymentConfig.java | 65 ---
.../flink/FlinkMiniClusterDeploymentConfig.java | 26 --
.../streampipes/wrapper/flink/FlinkRuntime.java | 355 ---------------
.../wrapper/flink/FlinkRuntimeProvider.java | 37 --
.../wrapper/flink/FlinkSpMiniCluster.java | 57 ---
.../wrapper/flink/IDataProcessorProgram.java | 28 --
.../wrapper/flink/IDataSinkProgram.java | 28 --
.../streampipes/wrapper/flink/IFlinkProgram.java | 39 --
.../wrapper/flink/consumer/JmsFlinkConsumer.java | 73 ---
.../wrapper/flink/consumer/MqttFlinkConsumer.java | 73 ---
.../flink/converter/EventToMapConverter.java | 34 --
.../flink/converter/MapToEventConverter.java | 51 ---
.../wrapper/flink/logger/StatisticLogger.java | 41 --
.../flink/serializer/ByteArrayDeserializer.java | 45 --
.../flink/serializer/ByteArraySerializer.java | 44 --
.../flink/serializer/SimpleJmsSerializer.java | 45 --
.../flink/serializer/SimpleKafkaSerializer.java | 38 --
.../wrapper/flink/sink/JmsFlinkProducer.java | 66 ---
.../wrapper/flink/sink/MqttFlinkProducer.java | 66 ---
327 files changed, 19057 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fdf8ae2076..486fdc9dc1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -109,13 +109,6 @@ jobs:
push: true
tags: ${{ env.DOCKERHUB_APACHE_REPO }}/extensions-all-jvm:${{ env.MVN_VERSION }}
-# - name: Build and Push Docker Image pipeline-elements-experimental-flink
-# uses: docker/build-push-action@v6
-# with:
-# context: ./streampipes-extensions/streampipes-pipeline-elements-experimental-flink
-# platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
-# push: true
-# tags: ${{ env.DOCKERHUB_APACHE_REPO }}/pipeline-elements-experimental-flink:${{ env.MVN_VERSION }}
- name: Build and Push Docker Image extensions-all-iiot
uses: docker/build-push-action@v6
diff --git a/LICENSE b/LICENSE
index e0cdfdb1ad..fe876f4222 100644
--- a/LICENSE
+++ b/LICENSE
@@ -252,13 +252,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-===============================================================
-
-This project includes some sources that are also licensed under the Apache
-License Version 2.0:
-
-All files in streampipes-extensions/streampipes-processors-transformation-flink/src/main/java/src/main/java/com/kohlschutter/boilerpipe (https://github.com/kohlschutter/boilerpipe)
-
================================================================
This project includes some sources that are licensed under the MIT License:
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/README.md b/archetypes/streampipes-archetype-pe-processors-flink/README.md
deleted file mode 100644
index 93e1508054..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-## StreamPipes Maven Archetype for Flink-based Data Processors
-
-### Usage
-
-```bash
-mvn archetype:generate \
- -DarchetypeGroupId=org.apache.streampipes \
- -DarchetypeArtifactId=streampipes-archetype-pe-processors-flink \
- -DarchetypeVersion=<VERSION> \
- -DgroupId=org.myorga \
- -DclassNamePrefix=MyFlinkProcessor \
- -DartifactId=my-processor-flink \
- -Dversion=<VERSION>
-```
-
-### Variables
-
-* classNamePrefix: Will be used as a prefix to name your controller & parameter classes
-* packageName: Will be used as the package name
-
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/pom.xml b/archetypes/streampipes-archetype-pe-processors-flink/pom.xml
deleted file mode 100644
index 3a7ade522e..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/pom.xml
+++ /dev/null
@@ -1,63 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-parent</artifactId>
- <version>0.99.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <artifactId>streampipes-archetype-pe-processors-flink</artifactId>
- <packaging>maven-archetype</packaging>
-
- <name>streampipes-archetype-pe-processors-flink</name>
-
- <build>
- <extensions>
- <extension>
- <groupId>org.apache.maven.archetype</groupId>
- <artifactId>archetype-packaging</artifactId>
- <version>3.2.1</version>
- </extension>
- </extensions>
-
- <pluginManagement>
- <plugins>
- <plugin>
- <artifactId>maven-archetype-plugin</artifactId>
- <version>3.2.1</version>
- </plugin>
- </plugins>
- </pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <propertyExpansion>
- checkstyle.config.base.path=${project.parent.basedir}/tools/maven
- </propertyExpansion>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/META-INF/maven/archetype-metadata.xml
deleted file mode 100644
index 8b8a6946c5..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/META-INF/maven/archetype-metadata.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<archetype-descriptor
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0 http://maven.apache.org/xsd/archetype-descriptor-1.0.0.xsd"
- name="streampipes-pe-sinks-standalone"
- xmlns="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <fileSets>
- <fileSet filtered="true" packaged="true" encoding="UTF-8">
- <directory>src/main/java</directory>
- <includes>
- <include>**/*.java</include>
- </includes>
- </fileSet>
- <fileSet filtered="true" packaged="true" encoding="UTF-8">
- <directory>src/test/java</directory>
- <includes>
- <include>**/*.java</include>
- </includes>
- </fileSet>
- <fileSet filtered="true" packaged="false" encoding="UTF-8">
- <directory/>
- <includes>
- <include>Dockerfile</include>
- </includes>
- </fileSet>
- <fileSet filtered="true" packaged="false" encoding="UTF-8">
- <directory>src/main/resources</directory>
- <includes>
- <include>**/*.en</include>
- <include>**/*.md</include>
- </includes>
- </fileSet>
- <fileSet filtered="false" packaged="false" >
- <directory>src/main/resources</directory>
- <includes>
- <include>**/*.png</include>
- </includes>
- </fileSet>
- <fileSet filtered="true" packaged="false" encoding="UTF-8">
- <directory/>
- <includes>
- <include>development/env</include>
- </includes>
- </fileSet>
- </fileSets>
- <requiredProperties>
- <requiredProperty key="classNamePrefix">
- <defaultValue>Example</defaultValue>
- </requiredProperty>
- <requiredProperty key="packageName">
- <defaultValue>example</defaultValue>
- </requiredProperty>
- </requiredProperties>
-</archetype-descriptor>
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile
deleted file mode 100644
index bfe9dfdfdb..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM eclipse-temurin:17-jre-alpine
-
-COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar
-
-ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/development/env b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/development/env
deleted file mode 100644
index 077ddab6fc..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/development/env
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Those parameters are used by IntelliJ to set the default parameters for development
-SP_PORT=8005
-SP_HOST=host.docker.internal
-SP_DEBUG=true
-SP_FLINK_DEBUG=true
\ No newline at end of file
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml
deleted file mode 100644
index e2314a6c0e..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml
+++ /dev/null
@@ -1,159 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>${groupId}</groupId>
- <artifactId>${artifactId}</artifactId>
- <version>${version}</version>
-
- <properties>
- <sp.version>0.99.0-SNAPSHOT</sp.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-service-extensions</artifactId>
- <version>${sp.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-wrapper-flink</artifactId>
- <version>${sp.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- <version>1.11.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-sdk-bundle</artifactId>
- <version>${sp.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-to-slf4j</artifactId>
- <version>2.8.2</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>2.0.6</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.2.4</version>
- <dependencies>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
- <version>3.0.1</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/spring.handlers</resource>
- </transformer>
- <transformer
- implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">
- <resource>META-INF/spring.factories</resource>
- </transformer>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/spring.schemas</resource>
- </transformer>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>reference.conf</resource>
- </transformer>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>${package}.Init</mainClass>
- </transformer>
- </transformers>
- <filters>
-
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
- <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml
- </exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.10.1</version>
- <configuration>
- <release>17</release>
- <encoding>UTF-8</encoding>
- </configuration>
- </plugin>
- </plugins>
- <finalName>${artifactId}</finalName>
- </build>
-</project>
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/Init.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/Init.java
deleted file mode 100644
index 1a53b264a8..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/Init.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-
-package ${package};
-
-import ${package}.config.ConfigKeys;
-
-import ${package}.pe.processor.${packageName}.${classNamePrefix}Controller;
-
-import org.apache.streampipes.extensions.management.model.SpServiceDefinition;
-import org.apache.streampipes.extensions.management.model.SpServiceDefinitionBuilder;
-import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory;
-import org.apache.streampipes.messaging.kafka.SpKafkaProtocolFactory;
-import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory;
-import org.apache.streampipes.messaging.pulsar.SpPulsarProtocolFactory;
-import org.apache.streampipes.messaging.nats.SpNatsProtocolFactory;
-import org.apache.streampipes.service.extensions.StreamPipesExtensionsServiceBase;
-
-public class Init extends StreamPipesExtensionsServiceBase {
-
- public static void main(String[] args) throws Exception {
- new Init().init();
- }
-
- @Override
- public SpServiceDefinition provideServiceDefinition() {
- return SpServiceDefinitionBuilder.create("${package}",
- "Apache Flink processor",
- "",
- 8090)
- .registerPipelineElement(new ${classNamePrefix}Controller())
- .registerMessagingProtocols(
- new SpKafkaProtocolFactory(),
- new SpJmsProtocolFactory(),
- new SpMqttProtocolFactory(),
- new SpNatsProtocolFactory()),
- new SpPulsarProtocolFactory())
- .addConfig(ConfigKeys.FLINK_HOST, "jobmanager", "Hostname of the Flink Jobmanager")
- .addConfig(ConfigKeys.FLINK_PORT, 8081, "Port of the Flink Jobmanager")
- .addConfig(ConfigKeys.DEBUG, false, "Debug/Mini cluster mode of Flink program")
- .addConfig(ConfigKeys.FLINK_JAR_FILE_LOC, "./streampipes-processing-element-container.jar", "Jar file location")
- .addConfig(ConfigKeys.SERVICE_NAME, "sp fft stream analytics metrics", "Data processor service name")
- .addConfig(ConfigKeys.HOST, "${artifactId}", "Data processor host")
- .build();
- }
-}
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java
deleted file mode 100644
index 0dc3eb95cf..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-
-package ${package}.config;
-
-public class ConfigKeys {
- public static final String HOST = "SP_HOST";
- public static final String PORT = "SP_PORT";
- public static final String SERVICE_NAME = "SP_SERVICE_NAME";
- public static final String FLINK_HOST = "SP_FLINK_HOST";
- public static final String FLINK_PORT = "SP_FLINK_PORT";
- public static final String FLINK_DEBUG = "SP_FLINK_DEBUG";
- public static final String DEBUG = "SP_FLINK_DEBUG";
- public static final String FLINK_JAR_FILE_LOC = "SP_FLINK_JAR_FILE_LOC";
-}
\ No newline at end of file
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__.java
deleted file mode 100644
index f61fc83a2f..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-
-package ${package}.pe.processor.${packageName};
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.util.Collector;
-
-import org.apache.streampipes.model.runtime.Event;
-
-public class ${classNamePrefix} implements FlatMapFunction<Event, Event> {
-
- private String exampleText;
-
- public ${classNamePrefix}(String exampleText) {
- this.exampleText = exampleText;
- }
-
- @Override
- public void flatMap(Event event, Collector<Event> out) {
- event.addField("appendedText", exampleText);
- out.collect(event);
- }
-}
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java
deleted file mode 100644
index de85a9d338..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-
-package ${package}.pe.processor.${packageName};
-
-import org.apache.streampipes.extensions.management.config.ConfigExtractor;
-import org.apache.streampipes.sdk.builder.PrimitivePropertyBuilder;
-import org.apache.streampipes.sdk.utils.Datatypes;
-import org.apache.streampipes.client.StreamPipesClient;
-import org.apache.streampipes.model.DataProcessorType;
-import org.apache.streampipes.model.graph.DataProcessorDescription;
-import org.apache.streampipes.model.graph.DataProcessorInvocation;
-import org.apache.streampipes.sdk.builder.ProcessingElementBuilder;
-import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
-import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
-import org.apache.streampipes.sdk.helpers.EpRequirements;
-import org.apache.streampipes.sdk.helpers.Labels;
-import org.apache.streampipes.sdk.helpers.OutputStrategies;
-import org.apache.streampipes.sdk.helpers.*;
-import org.apache.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
-import org.apache.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.apache.streampipes.model.extensions.ExtensionAssetType;
-
-public class ${classNamePrefix}Controller extends FlinkDataProcessorDeclarer<${classNamePrefix}Parameters> {
-
- private static final String EXAMPLE_KEY = "example-key";
-
- @Override
- public DataProcessorDescription declareModel() {
- return ProcessingElementBuilder.create("${package}.pe.processor.${packageName}")
- .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON)
- .withLocales(Locales.EN)
- .category(DataProcessorType.ENRICH)
- .requiredStream(StreamRequirementsBuilder
- .create()
- .requiredProperty(EpRequirements.anyProperty())
- .build())
- .requiredTextParameter(Labels.withId(EXAMPLE_KEY))
- .outputStrategy(OutputStrategies.append(
- PrimitivePropertyBuilder.create(
- Datatypes.String, "appendedText")
- .build()))
- .build();
- }
-
- @Override
- public FlinkDataProcessorRuntime<${classNamePrefix}Parameters> getRuntime(
- DataProcessorInvocation graph,
- ProcessingElementParameterExtractor extractor,
- ConfigExtractor configExtractor,
- StreamPipesClient streamPipesClient) {
-
- String exampleText = extractor.singleValueParameter(EXAMPLE_KEY, String.class);
- ${classNamePrefix}Parameters params = new ${classNamePrefix}Parameters(graph, exampleText);
-
- return new ${classNamePrefix}Program(params, configExtractor, streamPipesClient);
- }
-}
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Parameters.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Parameters.java
deleted file mode 100644
index d86ea3580d..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Parameters.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-
-package ${package}.pe.processor.${packageName};
-
-import org.apache.streampipes.model.graph.DataProcessorInvocation;
-import org.apache.streampipes.wrapper.params.binding.EventProcessorBindingParams;
-
-public class ${classNamePrefix}Parameters extends EventProcessorBindingParams {
-
- private String exampleText;
- private static final long serialVersionUID = 1L;
-
-
- public ${classNamePrefix}Parameters(DataProcessorInvocation graph, String exampleText) {
- super(graph);
- this.exampleText = exampleText;
- }
-
- public String getExampleText() {
- return exampleText;
- }
-}
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Program.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Program.java
deleted file mode 100644
index fcad195dc9..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Program.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-
-package ${package}.pe.processor.${packageName};
-
-import ${package}.config.ConfigKeys;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.streampipes.client.StreamPipesClient;
-import org.apache.streampipes.extensions.management.config.ConfigExtractor;
-import org.apache.streampipes.model.graph.DataProcessorInvocation;
-import org.apache.streampipes.model.runtime.Event;
-import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
-import org.apache.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.apache.streampipes.wrapper.flink.FlinkDeploymentConfig;
-import org.apache.streampipes.svcdiscovery.api.SpConfig;
-
-import ${package}.config.ConfigKeys;
-
-import java.io.Serializable;
-
-public class ${classNamePrefix}Program extends FlinkDataProcessorRuntime<${classNamePrefix}Parameters> implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private final ${classNamePrefix}Parameters params;
-
- public ${classNamePrefix}Program(${classNamePrefix}Parameters params,
- ConfigExtractor configExtractor,
- StreamPipesClient streamPipesClient)
- {
- super(params, configExtractor, streamPipesClient);
- this.params = params;
- }
-
-
- @Override
- protected FlinkDeploymentConfig getDeploymentConfig(ConfigExtractor configExtractor) {
- SpConfig config = configExtractor.getConfig();
- return new FlinkDeploymentConfig(config.getString(ConfigKeys.FLINK_JAR_FILE_LOC),
- config.getString(ConfigKeys.FLINK_HOST),
- config.getInteger(ConfigKeys.FLINK_PORT),
- config.getBoolean(ConfigKeys.DEBUG));
- }
-
- @Override
- protected DataStream<Event> getApplicationLogic(DataStream<Event>... dataStreams){
-
- return dataStreams[0]
- .flatMap(new ${classNamePrefix}(this.params.getExampleText()));
- }
-}
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.processor.__packageName__/documentation.md b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.processor.__packageName__/documentation.md
deleted file mode 100644
index 1e0e266f39..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.processor.__packageName__/documentation.md
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-#set( $double_pound = '##' )
-
-${double_pound} ${classNamePrefix}
-
-<p align="center">
- <img src="icon.png" width="150px;" class="pe-image-documentation"/>
-</p>
-
-***
-
-${double_pound} Description
-Describe your new data processor here!
-
-***
-
-${double_pound} Required input
-What are the input requirements of your data processor?
-
-***
-
-${double_pound} Configuration
-What are the configurations a user has to provide?
-
-${double_pound} Output
-How do the events your data processor emits look like?
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.processor.__packageName__/icon.png b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.processor.__packageName__/icon.png
deleted file mode 100644
index 05d4b1deea..0000000000
Binary files a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.processor.__packageName__/icon.png and /dev/null differ
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.processor.__packageName__/strings.en b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.processor.__packageName__/strings.en
deleted file mode 100644
index d23ce26ca9..0000000000
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.processor.__packageName__/strings.en
+++ /dev/null
@@ -1,5 +0,0 @@
-${package}.pe.processor.${packageName}.title=${classNamePrefix}
-${package}.pe.processor.${packageName}.description=Description of data processor
-
-example-key.title=Example Text Parameter
-example-key.description=Example Text Parameter Description
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/README.md b/archetypes/streampipes-archetype-pe-sinks-flink/README.md
deleted file mode 100644
index 4dfea82e65..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-## StreamPipes Maven Archetype for Flink-based Sinks
-
-### Usage
-
-```bash
-mvn archetype:generate \
- -DarchetypeGroupId=org.apache.streampipes \
- -DarchetypeArtifactId=streampipes-archetype-pe-sinks-flink \
- -DarchetypeVersion=<VERSION> \
- -DgroupId=org.myorga \
- -DclassNamePrefix=MyFlinkSink \
- -DartifactId=my-flink-sink\
- -Dversion=<VERSION>
-```
-
-### Variables
-
-* classNamePrefix: Will be used as a prefix to name your controller & parameter classes
-* packageName: Will be used as the package name
-
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml
deleted file mode 100644
index d130a42c07..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-parent</artifactId>
- <version>0.99.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <artifactId>streampipes-archetype-pe-sinks-flink</artifactId>
- <packaging>maven-archetype</packaging>
-
- <name>streampipes-archetype-pe-sinks-flink</name>
-
- <build>
- <extensions>
- <extension>
- <groupId>org.apache.maven.archetype</groupId>
- <artifactId>archetype-packaging</artifactId>
- <version>3.2.1</version>
- </extension>
- </extensions>
-
- <pluginManagement>
- <plugins>
- <plugin>
- <artifactId>maven-archetype-plugin</artifactId>
- <version>3.2.1</version>
- </plugin>
- </plugins>
- </pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <propertyExpansion>
- checkstyle.config.base.path=${project.parent.basedir}/tools/maven
- </propertyExpansion>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/META-INF/maven/archetype-metadata.xml
deleted file mode 100644
index 8b8a6946c5..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/META-INF/maven/archetype-metadata.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<archetype-descriptor
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0 http://maven.apache.org/xsd/archetype-descriptor-1.0.0.xsd"
- name="streampipes-pe-sinks-standalone"
- xmlns="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <fileSets>
- <fileSet filtered="true" packaged="true" encoding="UTF-8">
- <directory>src/main/java</directory>
- <includes>
- <include>**/*.java</include>
- </includes>
- </fileSet>
- <fileSet filtered="true" packaged="true" encoding="UTF-8">
- <directory>src/test/java</directory>
- <includes>
- <include>**/*.java</include>
- </includes>
- </fileSet>
- <fileSet filtered="true" packaged="false" encoding="UTF-8">
- <directory/>
- <includes>
- <include>Dockerfile</include>
- </includes>
- </fileSet>
- <fileSet filtered="true" packaged="false" encoding="UTF-8">
- <directory>src/main/resources</directory>
- <includes>
- <include>**/*.en</include>
- <include>**/*.md</include>
- </includes>
- </fileSet>
- <fileSet filtered="false" packaged="false" >
- <directory>src/main/resources</directory>
- <includes>
- <include>**/*.png</include>
- </includes>
- </fileSet>
- <fileSet filtered="true" packaged="false" encoding="UTF-8">
- <directory/>
- <includes>
- <include>development/env</include>
- </includes>
- </fileSet>
- </fileSets>
- <requiredProperties>
- <requiredProperty key="classNamePrefix">
- <defaultValue>Example</defaultValue>
- </requiredProperty>
- <requiredProperty key="packageName">
- <defaultValue>example</defaultValue>
- </requiredProperty>
- </requiredProperties>
-</archetype-descriptor>
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile
deleted file mode 100644
index bfe9dfdfdb..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM eclipse-temurin:17-jre-alpine
-
-COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar
-
-ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/development/env b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/development/env
deleted file mode 100644
index 077ddab6fc..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/development/env
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Those parameters are used by IntelliJ to set the default parameters for development
-SP_PORT=8005
-SP_HOST=host.docker.internal
-SP_DEBUG=true
-SP_FLINK_DEBUG=true
\ No newline at end of file
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml
deleted file mode 100644
index dfdf4f8405..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml
+++ /dev/null
@@ -1,160 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>${groupId}</groupId>
- <artifactId>${artifactId}</artifactId>
- <version>${version}</version>
-
- <properties>
- <sp.version>0.99.0-SNAPSHOT</sp.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-service-extensions</artifactId>
- <version>${sp.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-wrapper-flink</artifactId>
- <version>${sp.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- <version>1.11.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-sdk-bundle</artifactId>
- <version>${sp.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-to-slf4j</artifactId>
- <version>2.8.2</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>2.0.6</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.3</version>
- <dependencies>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
- <version>2.4.1</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/spring.handlers</resource>
- </transformer>
- <transformer
- implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">
- <resource>META-INF/spring.factories</resource>
- </transformer>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/spring.schemas</resource>
- </transformer>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>reference.conf</resource>
- </transformer>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>${package}.Init</mainClass>
- </transformer>
- </transformers>
- <filters>
-
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
- <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml
- </exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.1</version>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- <encoding>UTF-8</encoding>
- </configuration>
- </plugin>
- </plugins>
- <finalName>${artifactId}</finalName>
- </build>
-</project>
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/Init.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/Init.java
deleted file mode 100644
index 32f521f419..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/Init.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-package ${package};
-
-import org.apache.streampipes.extensions.management.model.SpServiceDefinition;
-import org.apache.streampipes.extensions.management.model.SpServiceDefinitionBuilder;
-import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory;
-import org.apache.streampipes.messaging.kafka.SpKafkaProtocolFactory;
-import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory;
-import org.apache.streampipes.messaging.pulsar.SpPulsarProtocolFactory;
-import org.apache.streampipes.messaging.nats.SpNatsProtocolFactory;
-import org.apache.streampipes.service.extensions.StreamPipesExtensionsServiceBase;
-
-import ${package}.config.ConfigKeys;
-import ${package}.pe.sink.${packageName}.${classNamePrefix}Controller;
-
-public class Init extends StreamPipesExtensionsServiceBase {
-
- public static void main(String[] args) throws Exception {
- new Init().init();
- }
-
- @Override
- public SpServiceDefinition provideServiceDefinition() {
- return SpServiceDefinitionBuilder.create("${package}",
- "Apache Flink sink",
- "",
- 8090)
- .registerPipelineElement(new ${classNamePrefix}Controller())
- .registerMessagingProtocols(
- new SpKafkaProtocolFactory(),
- new SpJmsProtocolFactory(),
- new SpMqttProtocolFactory(),
- new SpNatsProtocolFactory()),
- new SpPulsarProtocolFactory())
- .addConfig(ConfigKeys.FLINK_HOST, "jobmanager", "Hostname of the Flink Jobmanager")
- .addConfig(ConfigKeys.FLINK_PORT, 8081, "Port of the Flink Jobmanager")
- .addConfig(ConfigKeys.DEBUG, false, "Debug/Mini cluster mode of Flink program")
- .addConfig(ConfigKeys.FLINK_JAR_FILE_LOC, "./streampipes-processing-element-container.jar", "Jar file location")
- .addConfig(ConfigKeys.SERVICE_NAME, "sp fft stream analytics metrics", "Data processor service name")
- .addConfig(ConfigKeys.HOST, "${artifactId}", "Data processor host")
- .build();
- }
-}
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java
deleted file mode 100644
index a828f48101..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-package ${package}.config;
-
-public class ConfigKeys {
- public final static String HOST = "SP_HOST";
- public final static String PORT = "SP_PORT";
- public final static String SERVICE_NAME = "SP_SERVICE_NAME";
- public final static String FLINK_HOST = "SP_FLINK_HOST";
- public final static String FLINK_PORT = "SP_FLINK_PORT";
- public static final String DEBUG = "SP_FLINK_DEBUG";
- public static final String FLINK_JAR_FILE_LOC = "SP_FLINK_JAR_FILE_LOC";
-}
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java
deleted file mode 100644
index 3d15c73b7a..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-package ${package}.pe.sink.${packageName};
-
-import ${package}.config.Config;
-import org.apache.streampipes.client.StreamPipesClient;
-import org.apache.streampipes.extensions.management.config.ConfigExtractor;
-import org.apache.streampipes.model.DataSinkType;
-import org.apache.streampipes.model.graph.DataSinkDescription;
-import org.apache.streampipes.model.graph.DataSinkInvocation;
-import org.apache.streampipes.sdk.builder.DataSinkBuilder;
-import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
-import org.apache.streampipes.sdk.extractor.DataSinkParameterExtractor;
-import org.apache.streampipes.sdk.helpers.EpRequirements;
-import org.apache.streampipes.sdk.helpers.Labels;
-import org.apache.streampipes.wrapper.flink.FlinkDataSinkDeclarer;
-import org.apache.streampipes.wrapper.flink.FlinkDataSinkRuntime;
-import org.apache.streampipes.sdk.helpers.*;
-import org.apache.streampipes.model.extensions.ExtensionAssetType;
-
-public class ${classNamePrefix}Controller extends FlinkDataSinkDeclarer<${classNamePrefix}Parameters> {
-
- private static final String HOST_KEY = "host-key";
- private static final String PORT_KEY = "port-key";
- private static final String PASSWORD_KEY = "password-key";
-
- @Override
- public DataSinkDescription declareModel() {
- return DataSinkBuilder.create("${package}.pe.sink.${packageName}")
- .category(DataSinkType.NOTIFICATION)
- .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON)
- .withLocales(Locales.EN)
- .requiredStream(StreamRequirementsBuilder
- .create()
- .requiredProperty(EpRequirements.anyProperty())
- .build())
- .requiredTextParameter(Labels.withId(HOST_KEY))
- .requiredIntegerParameter(Labels.withId(PORT_KEY), 1234)
- .requiredSecret(Labels.withId(PASSWORD_KEY))
- .build();
- }
-
- @Override
- public FlinkDataSinkRuntime<${classNamePrefix}Parameters> getRuntime(DataSinkInvocation graph,
- DataSinkParameterExtractor extractor,
- ConfigExtractor configExtractor,
- StreamPipesClient streamPipesClient) {
-
- String host = extractor.singleValueParameter(HOST_KEY, String.class);
- int port = extractor.singleValueParameter(PORT_KEY, Integer.class);
- String password = extractor.secretValue(PASSWORD_KEY);
- ${classNamePrefix}Parameters params = new ${classNamePrefix}Parameters(graph, host, port, password);
-
- return new ${classNamePrefix}Program(params, configExtractor, streamPipesClient);
- }
-}
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Parameters.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Parameters.java
deleted file mode 100644
index ff464653ba..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Parameters.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-
-package ${package}.pe.sink.${packageName};
-
-import org.apache.streampipes.model.graph.DataSinkInvocation;
-import org.apache.streampipes.wrapper.params.binding.EventSinkBindingParams;
-
-public class ${classNamePrefix}Parameters extends EventSinkBindingParams {
-
- private String host;
- private int port;
- private String password;
-
- public ${classNamePrefix}Parameters(DataSinkInvocation graph, String host, int port, String password) {
- super(graph);
- this.host = host;
- this.port = port;
- this.password = password;
- }
-
- public String getHost() {
- return host;
- }
-
- public int getPort() {
- return port;
- }
-
- public String getPassword() {
- return password;
- }
-
-}
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Program.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Program.java
deleted file mode 100644
index 621a810b61..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Program.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#set( $symbol_pound = '#' )
-#set( $symbol_dollar = '$' )
-#set( $symbol_escape = '\' )
-package ${package}.pe.sink.${packageName};
-
-import ${package}.config.ConfigKeys;
-
-import org.apache.streampipes.client.StreamPipesClient;
-import org.apache.streampipes.extensions.management.config.ConfigExtractor;
-import org.apache.streampipes.model.runtime.Event;
-import org.apache.streampipes.svcdiscovery.api.SpConfig;
-import org.apache.streampipes.wrapper.flink.FlinkDataSinkRuntime;
-import org.apache.streampipes.wrapper.flink.FlinkDeploymentConfig;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-
-import java.io.Serializable;
-
-public class ${classNamePrefix}Program extends FlinkDataSinkRuntime<${classNamePrefix}Parameters>
-implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private final ${classNamePrefix}Parameters params;
-
- public ${classNamePrefix}Program(${classNamePrefix}Parameters params,
- ConfigExtractor configExtractor,
- StreamPipesClient streamPipesClient) {
- super(params, configExtractor, streamPipesClient);
- this.params = params;
- }
-
- @Override
- protected FlinkDeploymentConfig getDeploymentConfig(ConfigExtractor configExtractor) {
- SpConfig config = configExtractor.getConfig();
- return new FlinkDeploymentConfig(config.getString(ConfigKeys.FLINK_JAR_FILE_LOC),
- config.getString(ConfigKeys.FLINK_HOST),
- config.getInteger(ConfigKeys.FLINK_PORT),
- config.getBoolean(ConfigKeys.DEBUG));
- }
-
- @Override
- public void getSink(
- DataStream<Event>... convertedStream) {
-
- // TODO add logic here
- // params.getHost();
- // params.getPort();
- // params.getPassword();
- }
-}
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.sink.__packageName__/documentation.md b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.sink.__packageName__/documentation.md
deleted file mode 100644
index 2e51497064..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.sink.__packageName__/documentation.md
+++ /dev/null
@@ -1,39 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-#set( $double_pound = '##' )
-
-${double_pound} ${classNamePrefix}
-
-<p align="center">
- <img src="icon.png" width="150px;" class="pe-image-documentation"/>
-</p>
-
-***
-
-${double_pound} Description
-Describe your new data sink here!
-
-***
-
-${double_pound} Required input
-What are the input requirements of your data sink?
-
-***
-
-${double_pound} Configuration
-What are the configurations a user has to provide?
\ No newline at end of file
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.sink.__packageName__/icon.png b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.sink.__packageName__/icon.png
deleted file mode 100644
index 05d4b1deea..0000000000
Binary files a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.sink.__packageName__/icon.png and /dev/null differ
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.sink.__packageName__/strings.en b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.sink.__packageName__/strings.en
deleted file mode 100644
index 054d02f5e2..0000000000
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/resources/__package__.pe.sink.__packageName__/strings.en
+++ /dev/null
@@ -1,11 +0,0 @@
-${package}.pe.sink.${packageName}.title=${classNamePrefix}
-${package}.pe.sink.${packageName}.description=Description of data sink
-
-host-key.title=Host
-host-key.description=IP/DNS of service
-
-port-key.title=Port
-port-key.description=Port of service
-
-password-key.title=Password
-password-key.description=Password to connect
\ No newline at end of file
diff --git a/installer/cli/deploy/standalone/jobmanager/docker-compose.dev.yml b/installer/cli/deploy/standalone/jobmanager/docker-compose.dev.yml
deleted file mode 100644
index 6d86951de2..0000000000
--- a/installer/cli/deploy/standalone/jobmanager/docker-compose.dev.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-services:
- jobmanager:
- ports:
- - "8081:8081"
- - "6123:6123"
diff --git a/installer/cli/deploy/standalone/jobmanager/docker-compose.yml b/installer/cli/deploy/standalone/jobmanager/docker-compose.yml
deleted file mode 100644
index 3d765b2eeb..0000000000
--- a/installer/cli/deploy/standalone/jobmanager/docker-compose.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-services:
- jobmanager:
- image: fogsyio/flink:1.13.5-scala_2.11
- command: jobmanager
- environment:
- - JOB_MANAGER_RPC_ADDRESS=jobmanager
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
- networks:
- spnet:
-
-networks:
- spnet:
- external: true
diff --git a/installer/cli/deploy/standalone/pipeline-elements-all-flink/docker-compose.dev.yml b/installer/cli/deploy/standalone/pipeline-elements-all-flink/docker-compose.dev.yml
deleted file mode 100644
index 6d449f9f17..0000000000
--- a/installer/cli/deploy/standalone/pipeline-elements-all-flink/docker-compose.dev.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-services:
- pipeline-elements-all-flink:
- labels:
- org.apache.streampipes.dev.pipeline-elements-all-flink.service.description: "Dev pipeline-elements-all-flink service"
-# ports:
-# - "8098:8090"
diff --git a/installer/cli/deploy/standalone/pipeline-elements-all-flink/docker-compose.yml b/installer/cli/deploy/standalone/pipeline-elements-all-flink/docker-compose.yml
deleted file mode 100644
index e7c0d90df8..0000000000
--- a/installer/cli/deploy/standalone/pipeline-elements-all-flink/docker-compose.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-services:
- pipeline-elements-all-flink:
- image: ${SP_DOCKER_REGISTRY}/pipeline-elements-all-flink:${SP_VERSION}
- depends_on:
- - "jobmanager"
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
- networks:
- spnet:
-
-networks:
- spnet:
- external: true
diff --git a/installer/cli/deploy/standalone/taskmanager/docker-compose.dev.yml b/installer/cli/deploy/standalone/taskmanager/docker-compose.dev.yml
deleted file mode 100644
index fce4439c63..0000000000
--- a/installer/cli/deploy/standalone/taskmanager/docker-compose.dev.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-services:
- taskmanager:
- labels:
- org.apache.streampipes.dev.taskmanager.service.description: "Dev taskmanager service"
diff --git a/installer/cli/deploy/standalone/taskmanager/docker-compose.yml b/installer/cli/deploy/standalone/taskmanager/docker-compose.yml
deleted file mode 100644
index 2453357179..0000000000
--- a/installer/cli/deploy/standalone/taskmanager/docker-compose.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-services:
- taskmanager:
- image: fogsyio/flink:1.13.5-scala_2.11
- depends_on:
- - jobmanager
- command: taskmanager
- links:
- - "jobmanager:jobmanager"
- environment:
- - JOB_MANAGER_RPC_ADDRESS=jobmanager
- - TASK_MANAGER_NUMBER_OF_TASK_SLOTS=20
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
- networks:
- spnet:
-
-networks:
- spnet:
- external: true
diff --git a/installer/cli/environments/full-monitoring b/installer/cli/environments/full-monitoring
index 9a09a34e1a..ea0fde8ed1 100644
--- a/installer/cli/environments/full-monitoring
+++ b/installer/cli/environments/full-monitoring
@@ -22,7 +22,6 @@ jobmanager
taskmanager
kafka
influxdb
-pipeline-elements-all-flink
grafana
prometheus
prometheus-alertmanager
diff --git a/pom.xml b/pom.xml
index 87b3e4324a..ccd94266e7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,7 +62,6 @@
<error-prone.version>2.10.0</error-prone.version>
<file-management.version>3.1.0</file-management.version>
<findbugs.version>3.0.2</findbugs.version>
- <flink.version>1.13.5</flink.version>
<fogsy-qudt.version>1.0</fogsy-qudt.version>
<geojson-jackson.version>1.14</geojson-jackson.version>
<google-maps-services.version>2.2.0</google-maps-services.version>
@@ -442,21 +441,6 @@
<artifactId>commons-text</artifactId>
<version>${commons-text.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-kafka_2.11</artifactId>
- <version>${flink.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${flink.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.11</artifactId>
- <version>${flink.version}</version>
- </dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
@@ -1070,8 +1054,6 @@
<modules>
<module>archetypes/streampipes-archetype-extensions-jvm</module>
- <module>archetypes/streampipes-archetype-pe-sinks-flink</module>
- <module>archetypes/streampipes-archetype-pe-processors-flink</module>
<module>streampipes-client</module>
<module>streampipes-client-api</module>
<module>streampipes-commons</module>
@@ -1127,7 +1109,6 @@
<module>streampipes-vocabulary</module>
<module>streampipes-wrapper</module>
<module>streampipes-wrapper-distributed</module>
- <module>streampipes-wrapper-flink</module>
<module>streampipes-wrapper-kafka-streams</module>
<module>streampipes-wrapper-siddhi</module>
<module>streampipes-wrapper-standalone</module>
@@ -1471,15 +1452,10 @@
<exclude>streampipes-code-generation/src/test/resources/expected_web_xml</exclude>
<!-- Archetype resources should not have ASF license header -->
- <exclude>archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/**</exclude>
<exclude>archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/**</exclude>
- <exclude>archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/**</exclude>
<exclude>archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/**</exclude>
<!-- External files -->
- <exclude>
- streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/**
- </exclude>
<exclude>
streampipes-extensions/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/processor/revgeocoder/geocode/**
</exclude>
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
index 69b76952bd..5ad79d8e07 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
@@ -75,13 +75,6 @@ public enum Envs {
SP_TS_STORAGE_IOT_DB_USER("SP_TS_STORAGE_IOT_DB_USER", "root"),
SP_TS_STORAGE_IOT_DB_PASSWORD("SP_TS_STORAGE_IOT_DB_PASSWORD", "root"),
- SP_FLINK_JAR_FILE_LOC(
- "SP_FLINK_JAR_FILE_LOC", "./streampipes-processing-element-container.jar"),
-
- SP_FLINK_JOBMANAGER_HOST("SP_FLINK_JOBMANAGER_HOST", "jobmanager"),
-
- SP_FLINK_JOBMANAGER_PORT("SP_FLINK_JOBMANAGER_PORT", "8081"),
-
SP_PROMETHEUS_ENDPOINT_INCLUDE("SP_PROMETHEUS_ENDPOINT_INCLUDE", "health,prometheus"),
SP_SETUP_PROMETHEUS_ENDPOINT("SP_SETUP_PROMETHEUS_ENDPOINT", "false"),
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java
index b04aec1da8..3627c40299 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java
@@ -242,21 +242,6 @@ public class DefaultEnvironment implements Environment {
return new StringEnvironmentVariable(Envs.SP_EXT_ASSET_BASE_DIR);
}
- @Override
- public StringEnvironmentVariable getFlinkJarFileLoc() {
- return new StringEnvironmentVariable(Envs.SP_FLINK_JAR_FILE_LOC);
- }
-
- @Override
- public StringEnvironmentVariable getFlinkJobmanagerHost() {
- return new StringEnvironmentVariable(Envs.SP_FLINK_JOBMANAGER_HOST);
- }
-
- @Override
- public IntEnvironmentVariable getFlinkJobmanagerPort() {
- return new IntEnvironmentVariable(Envs.SP_FLINK_JOBMANAGER_PORT);
- }
-
@Override
public StringEnvironmentVariable getPrometheusEndpointInclude() {
return new StringEnvironmentVariable(Envs.SP_PROMETHEUS_ENDPOINT_INCLUDE);
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java
index 79cc859251..628f03ef40 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java
@@ -125,13 +125,6 @@ public interface Environment {
StringEnvironmentVariable getExtAssetBaseDir();
- // Flink Wrapper
- StringEnvironmentVariable getFlinkJarFileLoc();
-
- StringEnvironmentVariable getFlinkJobmanagerHost();
-
- IntEnvironmentVariable getFlinkJobmanagerPort();
-
// prometheus
StringEnvironmentVariable getPrometheusEndpointInclude();
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/Dockerfile b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/Dockerfile
deleted file mode 100644
index f0d2b39c96..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/Dockerfile
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM eclipse-temurin:17-jre-focal
-
-COPY target/streampipes-pipeline-elements-all-flink.jar /streampipes-processing-element-container.jar
-
-ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/development/env b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/development/env
deleted file mode 100644
index 077ddab6fc..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/development/env
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Those parameters are used by IntelliJ to set the default parameters for development
-SP_PORT=8005
-SP_HOST=host.docker.internal
-SP_DEBUG=true
-SP_FLINK_DEBUG=true
\ No newline at end of file
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/pom.xml b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/pom.xml
deleted file mode 100644
index fa07339259..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/pom.xml
+++ /dev/null
@@ -1,122 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streampipes-extensions</artifactId>
- <groupId>org.apache.streampipes</groupId>
- <version>0.99.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streampipes-pipeline-elements-experimental-flink</artifactId>
-
- <properties>
- <maven.deploy.skip>true</maven.deploy.skip>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-sdk-bundle</artifactId>
- <version>0.99.0-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-wrapper-flink</artifactId>
- <version>0.99.0-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-cep_2.11</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime_2.11</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-parser-combinators_2.11</artifactId>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- </dependency>
- <dependency>
- <groupId>net.sourceforge.nekohtml</groupId>
- <artifactId>nekohtml</artifactId>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- </dependency>
- <dependency>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/spring.handlers</resource>
- </transformer>
- <transformer implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">
- <resource>META-INF/spring.factories</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/spring.schemas</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>reference.conf</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.streampipes.pe.flink.AllFlinkPipelineElementsInit
- </mainClass>
- </transformer>
- </transformers>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- <finalName>streampipes-pipeline-elements-all-flink</finalName>
- </build>
-</project>
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeDocumentSource.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeDocumentSource.java
deleted file mode 100644
index 78b1545a81..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeDocumentSource.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe;
-
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * Something that can be represented as a {@link TextDocument}.
- */
-public interface BoilerpipeDocumentSource {
- TextDocument toTextDocument() throws BoilerpipeProcessingException;
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeExtractor.java
deleted file mode 100644
index 32fb11140c..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeExtractor.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe;
-
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import org.xml.sax.InputSource;
-
-import java.io.Reader;
-
-/**
- * Describes a complete filter pipeline.
- */
-public interface BoilerpipeExtractor extends BoilerpipeFilter {
- /**
- * Extracts text from the HTML code given as a String.
- *
- * @param html The HTML code as a String.
- * @return The extracted text.
- * @throws BoilerpipeProcessingException
- */
- public String getText(final String html) throws BoilerpipeProcessingException;
-
- /**
- * Extracts text from the HTML code available from the given {@link InputSource}.
- *
- * @param is The InputSource containing the HTML
- * @return The extracted text.
- * @throws BoilerpipeProcessingException
- */
- public String getText(final InputSource is) throws BoilerpipeProcessingException;
-
- /**
- * Extracts text from the HTML code available from the given {@link Reader}.
- *
- * @param r The Reader containing the HTML
- * @return The extracted text.
- * @throws BoilerpipeProcessingException
- */
- public String getText(final Reader r) throws BoilerpipeProcessingException;
-
- /**
- * Extracts text from the given {@link TextDocument} object.
- *
- * @param doc The {@link TextDocument}.
- * @return The extracted text.
- * @throws BoilerpipeProcessingException
- */
- public String getText(TextDocument doc) throws BoilerpipeProcessingException;
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeFilter.java
deleted file mode 100644
index 34bf0494ae..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeFilter.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe;
-
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.io.Serializable;
-
-/**
- * A generic {@link BoilerpipeFilter}. Takes a {@link TextDocument} and processes it somehow.
- */
-public interface BoilerpipeFilter extends Serializable {
- /**
- * Processes the given document <code>doc</code>.
- *
- * @param doc The {@link TextDocument} that is to be processed.
- * @return <code>true</code> if changes have been made to the {@link TextDocument}.
- * @throws BoilerpipeProcessingException
- */
- boolean process(final TextDocument doc) throws BoilerpipeProcessingException;
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeInput.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeInput.java
deleted file mode 100644
index fe85e92356..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeInput.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe;
-
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * A source that returns {@link TextDocument}s.
- */
-public interface BoilerpipeInput {
- /**
- * Returns (somehow) a {@link TextDocument}.
- *
- * @return A {@link TextDocument}.
- * @throws BoilerpipeProcessingException
- */
- TextDocument getTextDocument() throws BoilerpipeProcessingException;
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeProcessingException.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeProcessingException.java
deleted file mode 100644
index bc02ccb637..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/BoilerpipeProcessingException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe;
-
-/**
- * Exception for signaling failure in the processing pipeline.
- */
-public class BoilerpipeProcessingException extends Exception {
- private static final long serialVersionUID = 1L;
-
- public BoilerpipeProcessingException() {
- super();
- }
-
- public BoilerpipeProcessingException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public BoilerpipeProcessingException(String message) {
- super(message);
- }
-
- public BoilerpipeProcessingException(Throwable cause) {
- super(cause);
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/conditions/TextBlockCondition.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/conditions/TextBlockCondition.java
deleted file mode 100644
index cc0f684328..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/conditions/TextBlockCondition.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.conditions;
-
-import com.kohlschutter.boilerpipe.labels.ConditionalLabelAction;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-
-/**
- * Evaluates whether a given {@link TextBlock} meets a certain condition.
- *
- * Useful in combination with {@link ConditionalLabelAction}.
- */
-public interface TextBlockCondition {
- /**
- * Returns <code>true</code> iff the given {@link TextBlock} tb meets the defined condition.
- *
- * @param tb
- * @return <code><true</code> iff the condition is met.
- */
- boolean meetsCondition(final TextBlock tb);
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/Image.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/Image.java
deleted file mode 100644
index 9e88b9b63e..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/Image.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.document;
-
-/**
- * Represents an Image resource that is contained in the document.
- *
- * Any of the attributes may be null, except for "src".
- */
-public class Image implements Comparable<Image> {
- private final String src;
- private final String width;
- private final String height;
- private final String alt;
- private final int area;
-
- public Image(final String src, final String width, final String height, final String alt) {
- this.src = src;
- if (src == null) {
- throw new NullPointerException("src attribute must not be null");
- }
- this.width = nullTrim(width);
- this.height = nullTrim(height);
- this.alt = nullTrim(alt);
-
- if (width != null && height != null) {
- int a;
- try {
- a = Integer.parseInt(width) * Integer.parseInt(height);
- } catch (NumberFormatException e) {
- a = -1;
- }
- this.area = a;
- } else {
- this.area = -1;
- }
- }
-
- public String getSrc() {
- return src;
- }
-
- public String getWidth() {
- return width;
- }
-
- public String getHeight() {
- return height;
- }
-
- public String getAlt() {
- return alt;
- }
-
- private static String nullTrim(String s) {
- if (s == null) {
- return null;
- }
- s = s.trim();
- if (s.length() == 0) {
- return null;
- }
- return s;
- }
-
- /**
- * Returns the image's area (specified by width * height), or -1 if width/height weren't both
- * specified or could not be parsed.
- *
- * @return
- */
- public int getArea() {
- return area;
- }
-
- public String toString() {
- return src + "\twidth=" + width + "\theight=" + height + "\talt=" + alt + "\tarea=" + area;
- }
-
- @Override
- public int compareTo(Image o) {
- if (o == this) {
- return 0;
- }
- if (area > o.area) {
- return -1;
- } else if (area == o.area) {
- return src.compareTo(o.src);
- } else {
- return 1;
- }
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/TextBlock.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/TextBlock.java
deleted file mode 100644
index 51721b58ce..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/TextBlock.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.document;
-
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.BitSet;
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * Describes a block of text.
- *
- * A block can be an "atomic" text element (i.e., a sequence of text that is not interrupted by any
- * HTML markup) or a compound of such atomic elements.
- */
-public class TextBlock implements Cloneable {
- boolean isContent = false;
- private CharSequence text;
- Set<String> labels = null;
-
- int offsetBlocksStart;
- int offsetBlocksEnd;
-
- int numWords;
- int numWordsInAnchorText;
- int numWordsInWrappedLines;
- int numWrappedLines;
- float textDensity;
- float linkDensity;
-
- BitSet containedTextElements;
-
- private int numFullTextWords = 0;
- private int tagLevel;
-
- private static final BitSet EMPTY_BITSET = new BitSet();
- public static final TextBlock EMPTY_START = new TextBlock("", EMPTY_BITSET, 0, 0, 0, 0, -1);
- public static final TextBlock EMPTY_END = new TextBlock("", EMPTY_BITSET, 0, 0, 0, 0,
- Integer.MAX_VALUE);
-
- public TextBlock(final String text) {
- this(text, null, 0, 0, 0, 0, 0);
- }
-
- public TextBlock(final String text, final BitSet containedTextElements, final int numWords,
- final int numWordsInAnchorText, final int numWordsInWrappedLines, final int numWrappedLines,
- final int offsetBlocks) {
- this.text = text;
- this.containedTextElements = containedTextElements;
- this.numWords = numWords;
- this.numWordsInAnchorText = numWordsInAnchorText;
- this.numWordsInWrappedLines = numWordsInWrappedLines;
- this.numWrappedLines = numWrappedLines;
- this.offsetBlocksStart = offsetBlocks;
- this.offsetBlocksEnd = offsetBlocks;
- initDensities();
- }
-
- public boolean isContent() {
- return isContent;
- }
-
- public boolean setIsContent(boolean isContent) {
- if (isContent != this.isContent) {
- this.isContent = isContent;
- return true;
- } else {
- return false;
- }
- }
-
- public String getText() {
- return text.toString();
- }
-
- public int getNumWords() {
- return numWords;
- }
-
- public int getNumWordsInAnchorText() {
- return numWordsInAnchorText;
- }
-
- public float getTextDensity() {
- return textDensity;
- }
-
- public float getLinkDensity() {
- return linkDensity;
- }
-
- public void mergeNext(final TextBlock other) {
- if (!(text instanceof StringBuilder)) {
- text = new StringBuilder(text);
- }
- StringBuilder sb = (StringBuilder) text;
- sb.append('\n');
- sb.append(other.text);
-
- numWords += other.numWords;
- numWordsInAnchorText += other.numWordsInAnchorText;
-
- numWordsInWrappedLines += other.numWordsInWrappedLines;
- numWrappedLines += other.numWrappedLines;
-
- offsetBlocksStart = Math.min(offsetBlocksStart, other.offsetBlocksStart);
- offsetBlocksEnd = Math.max(offsetBlocksEnd, other.offsetBlocksEnd);
-
- initDensities();
-
- this.isContent |= other.isContent;
-
- if (containedTextElements == null) {
- containedTextElements = (BitSet) other.containedTextElements.clone();
- } else {
- containedTextElements.or(other.containedTextElements);
- }
-
- numFullTextWords += other.numFullTextWords;
-
- if (other.labels != null) {
- if (labels == null) {
- labels = new HashSet<String>(other.labels);
- } else {
- labels.addAll(other.labels);
- }
- }
-
- tagLevel = Math.min(tagLevel, other.tagLevel);
- }
-
- private void initDensities() {
- if (numWordsInWrappedLines == 0) {
- numWordsInWrappedLines = numWords;
- numWrappedLines = 1;
- }
- textDensity = numWordsInWrappedLines / (float) numWrappedLines;
- linkDensity = numWords == 0 ? 0 : numWordsInAnchorText / (float) numWords;
- }
-
- public int getOffsetBlocksStart() {
- return offsetBlocksStart;
- }
-
- public int getOffsetBlocksEnd() {
- return offsetBlocksEnd;
- }
-
- public String toString() {
- return "[" + offsetBlocksStart + "-" + offsetBlocksEnd + ";tl=" + tagLevel + "; nw=" + numWords
- + ";nwl=" + numWrappedLines + ";ld=" + linkDensity + "]\t"
- + (isContent ? "CONTENT" : "boilerplate") + "," + labels + "\n" + getText();
- }
-
- /**
- * Adds an arbitrary String label to this {@link TextBlock}.
- *
- * @param label The label
- * @see DefaultLabels
- */
- public void addLabel(final String label) {
- if (labels == null) {
- labels = new HashSet<String>(2);
- }
- labels.add(label);
- }
-
- /**
- * Checks whether this TextBlock has the given label.
- *
- * @param label The label
- * @return <code>true</code> if this block is marked by the given label.
- */
- public boolean hasLabel(final String label) {
- return labels != null && labels.contains(label);
- }
-
- public boolean removeLabel(final String label) {
- return labels != null && labels.remove(label);
- }
-
- /**
- * Returns the labels associated to this TextBlock, or <code>null</code> if no such labels exist.
- *
- * NOTE: The returned instance is the one used directly in TextBlock. You have full access to the
- * data structure. However it is recommended to use the label-specific methods in
- * {@link TextBlock} whenever possible.
- *
- * @return Returns the set of labels, or <code>null</code> if no labels was added yet.
- */
- public Set<String> getLabels() {
- return labels;
- }
-
- /**
- * Adds a set of labels to this {@link TextBlock}. <code>null</code>-references are silently
- * ignored.
- *
- * @param l The labels to be added.
- */
- public void addLabels(final Set<String> l) {
- if (l == null) {
- return;
- }
- if (this.labels == null) {
- this.labels = new HashSet<String>(l);
- } else {
- this.labels.addAll(l);
- }
- }
-
- /**
- * Adds a set of labels to this {@link TextBlock}. <code>null</code>-references are silently
- * ignored.
- *
- * @param l The labels to be added.
- */
- public void addLabels(final String... l) {
- if (l == null) {
- return;
- }
- if (this.labels == null) {
- this.labels = new HashSet<String>();
- }
- for (final String label : l) {
- this.labels.add(label);
- }
- }
-
- /**
- * Returns the containedTextElements BitSet, or <code>null</code>.
- *
- * @return
- */
- public BitSet getContainedTextElements() {
- return containedTextElements;
- }
-
- @Override
- protected TextBlock clone() {
- final TextBlock clone;
- try {
- clone = (TextBlock) super.clone();
- } catch (CloneNotSupportedException e) {
- throw new RuntimeException(e);
- }
- if (text != null && !(text instanceof String)) {
- clone.text = new StringBuilder(text);
- }
- if (labels != null && !labels.isEmpty()) {
- clone.labels = new HashSet<String>(labels);
- }
- if (containedTextElements != null) {
- clone.containedTextElements = (BitSet) containedTextElements.clone();
- }
-
- return clone;
- }
-
- public int getTagLevel() {
- return tagLevel;
- }
-
- public void setTagLevel(int tagLevel) {
- this.tagLevel = tagLevel;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/TextDocument.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/TextDocument.java
deleted file mode 100644
index 68eeca27f8..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/TextDocument.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.document;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * A text document, consisting of one or more {@link TextBlock}s.
- */
-public class TextDocument implements Cloneable {
- final List<TextBlock> textBlocks;
- String title;
-
- /**
- * Creates a new {@link TextDocument} with given {@link TextBlock}s, and no title.
- *
- * @param textBlocks The text blocks of this document.
- */
- public TextDocument(final List<TextBlock> textBlocks) {
- this(null, textBlocks);
- }
-
- /**
- * Creates a new {@link TextDocument} with given {@link TextBlock}s and given title.
- *
- * @param title The "main" title for this text document.
- * @param textBlocks The text blocks of this document.
- */
- public TextDocument(final String title, final List<TextBlock> textBlocks) {
- this.title = title;
- this.textBlocks = textBlocks;
- }
-
- /**
- * Returns the {@link TextBlock}s of this document.
- *
- * @return A list of {@link TextBlock}s, in sequential order of appearance.
- */
- public List<TextBlock> getTextBlocks() {
- return textBlocks;
- }
-
- /**
- * Returns the "main" title for this document, or <code>null</code> if no such title has ben set.
- *
- * @return The "main" title.
- */
- public String getTitle() {
- return title;
- }
-
- /**
- * Updates the "main" title for this document.
- *
- * @param title
- */
- public void setTitle(final String title) {
- this.title = title;
- }
-
- /**
- * Returns the {@link TextDocument}'s content.
- *
- * @return The content text.
- */
- public String getContent() {
- return getText(true, false);
- }
-
- /**
- * Returns the {@link TextDocument}'s content, non-content or both
- *
- * @param includeContent Whether to include TextBlocks marked as "content".
- * @param includeNonContent Whether to include TextBlocks marked as "non-content".
- * @return The text.
- */
- public String getText(boolean includeContent, boolean includeNonContent) {
- StringBuilder sb = new StringBuilder();
- LOOP:
- for (TextBlock block : getTextBlocks()) {
- if (block.isContent()) {
- if (!includeContent) {
- continue LOOP;
- }
- } else {
- if (!includeNonContent) {
- continue LOOP;
- }
- }
- sb.append(block.getText());
- sb.append('\n');
- }
- return sb.toString();
- }
-
- /**
- * Returns detailed debugging information about the contained {@link TextBlock}s.
- *
- * @return Debug information.
- */
- public String debugString() {
- StringBuilder sb = new StringBuilder();
- for (TextBlock tb : getTextBlocks()) {
- sb.append(tb.toString());
- sb.append('\n');
- }
- return sb.toString();
- }
-
- public TextDocument clone() {
- final List<TextBlock> list = new ArrayList<TextBlock>(textBlocks.size());
- for (TextBlock tb : textBlocks) {
- list.add(tb.clone());
- }
- return new TextDocument(title, list);
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/TextDocumentStatistics.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/TextDocumentStatistics.java
deleted file mode 100644
index 4f408125f5..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/TextDocumentStatistics.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.document;
-
-/**
- * Provides shallow statistics on a given {@link TextDocument}
- */
-public final class TextDocumentStatistics {
- private int numWords = 0;
- private int numBlocks = 0;
-
- /**
- * Computes statistics on a given {@link TextDocument}.
- *
- * @param doc The {@link TextDocument}.
- * @param contentOnly if true then o
- */
- public TextDocumentStatistics(final TextDocument doc, final boolean contentOnly) {
- for (TextBlock tb : doc.getTextBlocks()) {
- if (contentOnly && !tb.isContent()) {
- continue;
- }
-
- numWords += tb.getNumWords();
- numBlocks++;
- }
- }
-
- /**
- * Returns the average number of words at block-level (= overall number of words divided by the
- * number of blocks).
- *
- * @return Average
- */
- public float avgNumWords() {
- return numWords / (float) numBlocks;
- }
-
- /**
- * Returns the overall number of words in all blocks.
- *
- * @return Sum
- */
- public int getNumWords() {
- return numWords;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/package-info.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/package-info.java
deleted file mode 100644
index f8e67a733e..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/document/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * The Boilerpipe document model.
- */
-package com.kohlschutter.boilerpipe.document;
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/estimators/SimpleEstimator.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/estimators/SimpleEstimator.java
deleted file mode 100644
index 07fc869393..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/estimators/SimpleEstimator.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.estimators;
-
-import com.kohlschutter.boilerpipe.BoilerpipeExtractor;
-import com.kohlschutter.boilerpipe.extractors.ArticleExtractor;
-import com.kohlschutter.boilerpipe.extractors.DefaultExtractor;
-import com.kohlschutter.boilerpipe.document.TextDocumentStatistics;
-
-/**
- * Estimates the "goodness" of a {@link BoilerpipeExtractor} on a given document.
- */
-public final class SimpleEstimator {
-
- /**
- * Returns the singleton instance of {@link SimpleEstimator}
- */
- public static final SimpleEstimator INSTANCE = new SimpleEstimator();
-
- private SimpleEstimator() {
- }
-
- /**
- * Given the statistics of the document before and after applying the {@link BoilerpipeExtractor},
- * can we regard the extraction quality (too) low?
- *
- * Works well with {@link DefaultExtractor}, {@link ArticleExtractor} and others.
- *
- * @param dsBefore
- * @param dsAfter
- * @return true if low quality is to be expected.
- */
- public boolean isLowQuality(final TextDocumentStatistics dsBefore,
- final TextDocumentStatistics dsAfter) {
- if (dsBefore.getNumWords() < 90 || dsAfter.getNumWords() < 70) {
- return true;
- }
-
- if (dsAfter.avgNumWords() < 25) {
- return true;
- }
-
- return false;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/ArticleExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/ArticleExtractor.java
deleted file mode 100644
index 00cf8cc003..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/ArticleExtractor.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.filters.english.IgnoreBlocksAfterContentFilter;
-import com.kohlschutter.boilerpipe.filters.english.NumWordsRulesClassifier;
-import com.kohlschutter.boilerpipe.filters.english.TerminatingBlocksFinder;
-import com.kohlschutter.boilerpipe.filters.simple.BoilerplateBlockFilter;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.filters.heuristics.BlockProximityFusion;
-import com.kohlschutter.boilerpipe.filters.heuristics.DocumentTitleMatchClassifier;
-import com.kohlschutter.boilerpipe.filters.heuristics.ExpandTitleToContentFilter;
-import com.kohlschutter.boilerpipe.filters.heuristics.KeepLargestBlockFilter;
-import com.kohlschutter.boilerpipe.filters.heuristics.LargeBlockSameTagLevelToContentFilter;
-import com.kohlschutter.boilerpipe.filters.heuristics.ListAtEndFilter;
-import com.kohlschutter.boilerpipe.filters.heuristics.TrailingHeadlineToBoilerplateFilter;
-
-/**
- * A full-text extractor which is tuned towards news articles. In this scenario it achieves higher
- * accuracy than {@link DefaultExtractor}.
- */
-public final class ArticleExtractor extends ExtractorBase {
- public static final ArticleExtractor INSTANCE = new ArticleExtractor();
-
- /**
- * Returns the singleton instance for {@link ArticleExtractor}.
- */
- public static ArticleExtractor getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- return
-
- TerminatingBlocksFinder.INSTANCE.process(doc)
- | new DocumentTitleMatchClassifier(doc.getTitle()).process(doc)
- | NumWordsRulesClassifier.INSTANCE.process(doc)
- | IgnoreBlocksAfterContentFilter.DEFAULT_INSTANCE.process(doc)
- | TrailingHeadlineToBoilerplateFilter.INSTANCE.process(doc)
- | BlockProximityFusion.MAX_DISTANCE_1.process(doc)
- | BoilerplateBlockFilter.INSTANCE_KEEP_TITLE.process(doc)
- | BlockProximityFusion.MAX_DISTANCE_1_CONTENT_ONLY_SAME_TAGLEVEL.process(doc)
- | KeepLargestBlockFilter.INSTANCE_EXPAND_TO_SAME_TAGLEVEL_MIN_WORDS.process(doc)
- | ExpandTitleToContentFilter.INSTANCE.process(doc)
- | LargeBlockSameTagLevelToContentFilter.INSTANCE.process(doc)
- | ListAtEndFilter.INSTANCE.process(doc);
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/ArticleSentencesExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/ArticleSentencesExtractor.java
deleted file mode 100644
index e909acae61..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/ArticleSentencesExtractor.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.filters.simple.MinClauseWordsFilter;
-import com.kohlschutter.boilerpipe.filters.simple.SplitParagraphBlocksFilter;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * A full-text extractor which is tuned towards extracting sentences from news articles.
- */
-public final class ArticleSentencesExtractor extends ExtractorBase {
- public static final ArticleSentencesExtractor INSTANCE = new ArticleSentencesExtractor();
-
- /**
- * Returns the singleton instance for {@link ArticleSentencesExtractor}.
- */
- public static ArticleSentencesExtractor getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- return
-
- ArticleExtractor.INSTANCE.process(doc) | SplitParagraphBlocksFilter.INSTANCE.process(doc)
- | MinClauseWordsFilter.INSTANCE.process(doc);
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/CanolaExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/CanolaExtractor.java
deleted file mode 100644
index 0012e068eb..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/CanolaExtractor.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.estimators.SimpleEstimator;
-
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * A full-text extractor trained on <a href="http://krdwrd.org/">krdwrd</a> <a href
- * ="https://krdwrd.org/trac/attachment/wiki/Corpora/Canola/CANOLA.pdf">Canola </a>. Works well with
- * {@link SimpleEstimator}, too.
- */
-public class CanolaExtractor extends ExtractorBase {
- public static final CanolaExtractor INSTANCE = new CanolaExtractor();
-
- /**
- * Returns the singleton instance for {@link CanolaExtractor}.
- */
- public static CanolaExtractor getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
-
- return CLASSIFIER.process(doc);
- }
-
- /**
- * The actual classifier, exposed.
- */
- public static final BoilerpipeFilter CLASSIFIER = new BoilerpipeFilter() {
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- boolean hasChanges = false;
-
- ListIterator<TextBlock> it = textBlocks.listIterator();
- if (!it.hasNext()) {
- return false;
- }
- TextBlock prevBlock = TextBlock.EMPTY_START;
- TextBlock currentBlock = it.next();
- TextBlock nextBlock = it.hasNext() ? it.next() : TextBlock.EMPTY_START;
-
- hasChanges = classify(prevBlock, currentBlock, nextBlock) | hasChanges;
-
- if (nextBlock != TextBlock.EMPTY_START) {
- while (it.hasNext()) {
- prevBlock = currentBlock;
- currentBlock = nextBlock;
- nextBlock = it.next();
- hasChanges = classify(prevBlock, currentBlock, nextBlock) | hasChanges;
- }
- prevBlock = currentBlock;
- currentBlock = nextBlock;
- nextBlock = TextBlock.EMPTY_START;
- hasChanges = classify(prevBlock, currentBlock, nextBlock) | hasChanges;
- }
-
- return hasChanges;
- }
-
- protected boolean classify(final TextBlock prev, final TextBlock curr, final TextBlock next) {
- final boolean isContent =
- (curr.getLinkDensity() > 0 && next.getNumWords() > 11)
- || (curr.getNumWords() > 19 || (next.getNumWords() > 6 && next.getLinkDensity() == 0
- && prev.getLinkDensity() == 0 && (curr.getNumWords() > 6
- || prev.getNumWords() > 7 || next.getNumWords() > 19)));
-
- return curr.setIsContent(isContent);
- }
- };
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/CommonExtractors.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/CommonExtractors.java
deleted file mode 100644
index d2fb5abaa6..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/CommonExtractors.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeExtractor;
-
-/**
- * Provides quick access to common {@link BoilerpipeExtractor}s.
- */
-public final class CommonExtractors {
- private CommonExtractors() {
- }
-
- /**
- * Works very well for most types of Article-like HTML.
- */
- public static final ArticleExtractor ARTICLE_EXTRACTOR = ArticleExtractor.INSTANCE;
-
- /**
- * Usually worse than {@link ArticleExtractor}, but simpler/no heuristics.
- */
- public static final DefaultExtractor DEFAULT_EXTRACTOR = DefaultExtractor.INSTANCE;
-
- /**
- * Like {@link DefaultExtractor}, but keeps the largest text block only.
- */
- public static final LargestContentExtractor LARGEST_CONTENT_EXTRACTOR =
- LargestContentExtractor.INSTANCE;
-
- /**
- * Trained on krdwrd Canola (different definition of "boilerplate"). You may give it a try.
- */
- public static final CanolaExtractor CANOLA_EXTRACTOR = CanolaExtractor.INSTANCE;
-
- /**
- * Dummy Extractor; should return the input text. Use this to double-check that your problem is
- * within a particular {@link BoilerpipeExtractor}, or somewhere else.
- */
- public static final KeepEverythingExtractor KEEP_EVERYTHING_EXTRACTOR =
- KeepEverythingExtractor.INSTANCE;
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/DefaultExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/DefaultExtractor.java
deleted file mode 100644
index 6063781a53..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/DefaultExtractor.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.filters.english.DensityRulesClassifier;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.filters.heuristics.BlockProximityFusion;
-import com.kohlschutter.boilerpipe.filters.heuristics.SimpleBlockFusionProcessor;
-
-/**
- * A quite generic full-text extractor.
- */
-public class DefaultExtractor extends ExtractorBase {
- public static final DefaultExtractor INSTANCE = new DefaultExtractor();
-
- /**
- * Returns the singleton instance for {@link DefaultExtractor}.
- */
- public static DefaultExtractor getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
-
- return
-
- SimpleBlockFusionProcessor.INSTANCE.process(doc)
- | BlockProximityFusion.MAX_DISTANCE_1.process(doc)
- | DensityRulesClassifier.INSTANCE.process(doc);
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/ExtractorBase.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/ExtractorBase.java
deleted file mode 100644
index e74a6f18d7..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/ExtractorBase.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeExtractor;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.sax.BoilerpipeSAXInput;
-import com.kohlschutter.boilerpipe.sax.HTMLFetcher;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-import java.net.URL;
-
-/**
- * The base class of Extractors. Also provides some helper methods to quickly retrieve the text that
- * remained after processing.
- */
-public abstract class ExtractorBase implements BoilerpipeExtractor {
-
- /**
- * Extracts text from the HTML code given as a String.
- *
- * @param html The HTML code as a String.
- * @return The extracted text.
- * @throws BoilerpipeProcessingException
- */
- public String getText(final String html) throws BoilerpipeProcessingException {
- try {
- return getText(new BoilerpipeSAXInput(new InputSource(new StringReader(html)))
- .getTextDocument());
- } catch (SAXException e) {
- throw new BoilerpipeProcessingException(e);
- }
- }
-
- /**
- * Extracts text from the HTML code available from the given {@link InputSource}.
- *
- * @param is The InputSource containing the HTML
- * @return The extracted text.
- * @throws BoilerpipeProcessingException
- */
- public String getText(final InputSource is) throws BoilerpipeProcessingException {
- try {
- return getText(new BoilerpipeSAXInput(is).getTextDocument());
- } catch (SAXException e) {
- throw new BoilerpipeProcessingException(e);
- }
- }
-
- /**
- * Extracts text from the HTML code available from the given {@link URL}. NOTE: This method is
- * mainly to be used for show case purposes. If you are going to crawl the Web, consider using
- * {@link #getText(InputSource)} instead.
- *
- * @param url The URL pointing to the HTML code.
- * @return The extracted text.
- * @throws BoilerpipeProcessingException
- */
- public String getText(final URL url) throws BoilerpipeProcessingException {
- try {
- return getText(HTMLFetcher.fetch(url).toInputSource());
- } catch (IOException e) {
- throw new BoilerpipeProcessingException(e);
- }
- }
-
- /**
- * Extracts text from the HTML code available from the given {@link Reader}.
- *
- * @param r The Reader containing the HTML
- * @return The extracted text.
- * @throws BoilerpipeProcessingException
- */
- public String getText(final Reader r) throws BoilerpipeProcessingException {
- return getText(new InputSource(r));
- }
-
- /**
- * Extracts text from the given {@link TextDocument} object.
- *
- * @param doc The {@link TextDocument}.
- * @return The extracted text.
- * @throws BoilerpipeProcessingException
- */
- public String getText(TextDocument doc) throws BoilerpipeProcessingException {
- process(doc);
- return doc.getContent();
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/KeepEverythingExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/KeepEverythingExtractor.java
deleted file mode 100644
index 02f0574bde..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/KeepEverythingExtractor.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.filters.simple.MarkEverythingContentFilter;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * Marks everything as content.
- */
-public final class KeepEverythingExtractor extends ExtractorBase {
-
- public static final KeepEverythingExtractor INSTANCE = new KeepEverythingExtractor();
-
- private KeepEverythingExtractor() {
-
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- return MarkEverythingContentFilter.INSTANCE.process(doc);
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/KeepEverythingWithMinKWordsExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/KeepEverythingWithMinKWordsExtractor.java
deleted file mode 100644
index cd00e47ab2..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/KeepEverythingWithMinKWordsExtractor.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.filters.simple.MarkEverythingContentFilter;
-import com.kohlschutter.boilerpipe.filters.simple.MinWordsFilter;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.filters.heuristics.SimpleBlockFusionProcessor;
-
-/**
- * A full-text extractor which extracts the largest text component of a page. For news articles, it
- * may perform better than the {@link DefaultExtractor}, but usually worse than
- * {@link ArticleExtractor}.
- */
-public final class KeepEverythingWithMinKWordsExtractor extends ExtractorBase {
-
- private final MinWordsFilter filter;
-
- public KeepEverythingWithMinKWordsExtractor(final int kMin) {
- this.filter = new MinWordsFilter(kMin);
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- return SimpleBlockFusionProcessor.INSTANCE.process(doc)
- | MarkEverythingContentFilter.INSTANCE.process(doc) | filter.process(doc);
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/LargestContentExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/LargestContentExtractor.java
deleted file mode 100644
index c132f40f14..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/LargestContentExtractor.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.filters.english.NumWordsRulesClassifier;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.filters.heuristics.BlockProximityFusion;
-import com.kohlschutter.boilerpipe.filters.heuristics.KeepLargestBlockFilter;
-
-/**
- * A full-text extractor which extracts the largest text component of a page. For news articles, it
- * may perform better than the {@link DefaultExtractor}, but usually worse than
- * {@link ArticleExtractor}.
- */
-public final class LargestContentExtractor extends ExtractorBase {
- public static final LargestContentExtractor INSTANCE = new LargestContentExtractor();
-
- private LargestContentExtractor() {
- }
-
- /**
- * Returns the singleton instance for {@link LargestContentExtractor}.
- */
- public static LargestContentExtractor getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- return NumWordsRulesClassifier.INSTANCE.process(doc)
- | BlockProximityFusion.MAX_DISTANCE_1.process(doc)
- | KeepLargestBlockFilter.INSTANCE.process(doc);
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/NumWordsRulesExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/NumWordsRulesExtractor.java
deleted file mode 100644
index fdd12af2bf..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/NumWordsRulesExtractor.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.extractors;
-
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.filters.english.NumWordsRulesClassifier;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * A quite generic full-text extractor solely based upon the number of words per block (the current,
- * the previous and the next block).
- */
-public class NumWordsRulesExtractor extends ExtractorBase {
- public static final NumWordsRulesExtractor INSTANCE = new NumWordsRulesExtractor();
-
- /**
- * Returns the singleton instance for {@link NumWordsRulesExtractor}.
- */
- public static NumWordsRulesExtractor getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
-
- return NumWordsRulesClassifier.INSTANCE.process(doc);
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/package-info.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/package-info.java
deleted file mode 100644
index 97e389fe30..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/extractors/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * Some standard extractors (i.e., completely piped BoilerpipeFilters)
- */
-package com.kohlschutter.boilerpipe.extractors;
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/debug/PrintDebugFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/debug/PrintDebugFilter.java
deleted file mode 100644
index 183573738b..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/debug/PrintDebugFilter.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.debug;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.io.PrintWriter;
-
-/**
- * Prints debug information about the current state of the TextDocument. (= calls
- * {@link TextDocument#debugString()}.
- */
-public final class PrintDebugFilter implements BoilerpipeFilter {
- /**
- * Returns the default instance for {@link PrintDebugFilter}, which dumps debug information to
- * <code>System.out</code>
- */
- public static final PrintDebugFilter INSTANCE = new PrintDebugFilter(new PrintWriter(System.out,
- true));
- private final PrintWriter out;
-
- /**
- * Returns the default instance for {@link PrintDebugFilter}, which dumps debug information to
- * <code>System.out</code>
- */
- public static PrintDebugFilter getInstance() {
- return INSTANCE;
- }
-
- /**
- * Creates a new instance of {@link PrintDebugFilter}.
- *
- * Only use this method if you are not going to dump the debug information to
- * <code>System.out</code> -- for this case, use {@link #getInstance()} instead.
- *
- * @param out The target {@link PrintWriter}. Will not be closed
- */
- public PrintDebugFilter(final PrintWriter out) {
- this.out = out;
-
- }
-
- @Override
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- out.println(doc.debugString());
-
- return false;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/DensityRulesClassifier.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/DensityRulesClassifier.java
deleted file mode 100644
index b38fa1dd69..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/DensityRulesClassifier.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.english;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Classifies {@link TextBlock}s as content/not-content through rules that have been determined
- * using the C4.8 machine learning algorithm, as described in the paper
- * "Boilerplate Detection using Shallow Text Features", particularly using text densities and link
- * densities.
- */
-public class DensityRulesClassifier implements BoilerpipeFilter {
- public static final DensityRulesClassifier INSTANCE = new DensityRulesClassifier();
-
- /**
- * Returns the singleton instance for RulebasedBoilerpipeClassifier.
- */
- public static DensityRulesClassifier getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- boolean hasChanges = false;
-
- ListIterator<TextBlock> it = textBlocks.listIterator();
- if (!it.hasNext()) {
- return false;
- }
- TextBlock prevBlock = TextBlock.EMPTY_START;
- TextBlock currentBlock = it.next();
- TextBlock nextBlock = it.hasNext() ? it.next() : TextBlock.EMPTY_START;
-
- hasChanges = classify(prevBlock, currentBlock, nextBlock) | hasChanges;
-
- if (nextBlock != TextBlock.EMPTY_START) {
- while (it.hasNext()) {
- prevBlock = currentBlock;
- currentBlock = nextBlock;
- nextBlock = it.next();
- hasChanges = classify(prevBlock, currentBlock, nextBlock) | hasChanges;
- }
- prevBlock = currentBlock;
- currentBlock = nextBlock;
- nextBlock = TextBlock.EMPTY_START;
- hasChanges = classify(prevBlock, currentBlock, nextBlock) | hasChanges;
- }
-
- return hasChanges;
- }
-
- protected boolean classify(final TextBlock prev, final TextBlock curr, final TextBlock next) {
- final boolean isContent;
-
- if (curr.getLinkDensity() <= 0.333333) {
- if (prev.getLinkDensity() <= 0.555556) {
- if (curr.getTextDensity() <= 9) {
- if (next.getTextDensity() <= 10) {
- if (prev.getTextDensity() <= 4) {
- isContent = false;
- } else {
- isContent = true;
- }
- } else {
- isContent = true;
- }
- } else {
- if (next.getTextDensity() == 0) {
- isContent = false;
- } else {
- isContent = true;
- }
- }
- } else {
- if (next.getTextDensity() <= 11) {
- isContent = false;
- } else {
- isContent = true;
- }
- }
- } else {
- isContent = false;
- }
-
- return curr.setIsContent(isContent);
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/HeuristicFilterBase.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/HeuristicFilterBase.java
deleted file mode 100644
index 4aa17a0cfa..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/HeuristicFilterBase.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.english;
-
-import com.kohlschutter.boilerpipe.document.TextBlock;
-
-/**
- * Base class for some heuristics that are used by boilerpipe filters.
- */
-abstract class HeuristicFilterBase {
-
- protected static int getNumFullTextWords(final TextBlock tb) {
- return getNumFullTextWords(tb, 9);
- }
-
- protected static int getNumFullTextWords(final TextBlock tb, float minTextDensity) {
- if (tb.getTextDensity() >= minTextDensity) {
- return tb.getNumWords();
- } else {
- return 0;
- }
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/IgnoreBlocksAfterContentFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/IgnoreBlocksAfterContentFilter.java
deleted file mode 100644
index 0e4a35ad65..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/IgnoreBlocksAfterContentFilter.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.english;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.Iterator;
-
-/**
- * Marks all blocks as "non-content" that occur after blocks that have been marked
- * {@link DefaultLabels#INDICATES_END_OF_TEXT}. These marks are ignored unless a minimum number of
- * words in content blocks occur before this mark (default: 60). This can be used in conjunction
- * with an upstream {@link TerminatingBlocksFinder}.
- *
- * @see TerminatingBlocksFinder
- */
-public final class IgnoreBlocksAfterContentFilter extends HeuristicFilterBase implements
- BoilerpipeFilter {
- public static final IgnoreBlocksAfterContentFilter DEFAULT_INSTANCE =
- new IgnoreBlocksAfterContentFilter(60);
- public static final IgnoreBlocksAfterContentFilter INSTANCE_200 =
- new IgnoreBlocksAfterContentFilter(200);
- private final int minNumWords;
-
- /**
- * Returns the singleton instance for DeleteBlocksAfterContentFilter.
- */
- public static IgnoreBlocksAfterContentFilter getDefaultInstance() {
- return DEFAULT_INSTANCE;
- }
-
- public IgnoreBlocksAfterContentFilter(final int minNumWords) {
- this.minNumWords = minNumWords;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- boolean changes = false;
-
- int numWords = 0;
- boolean foundEndOfText = false;
- for (Iterator<TextBlock> it = doc.getTextBlocks().iterator(); it.hasNext(); ) {
- TextBlock block = it.next();
-
- final boolean endOfText = block.hasLabel(DefaultLabels.INDICATES_END_OF_TEXT);
- if (block.isContent()) {
- numWords += getNumFullTextWords(block);
- }
- if (endOfText && numWords >= minNumWords) {
- foundEndOfText = true;
- }
- if (foundEndOfText) {
- changes = true;
- block.setIsContent(false);
- }
- }
-
- return changes;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/IgnoreBlocksAfterContentFromEndFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/IgnoreBlocksAfterContentFromEndFilter.java
deleted file mode 100644
index e0fea3e82b..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/IgnoreBlocksAfterContentFromEndFilter.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * boilerpipe
- *
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- *
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.english;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Marks all blocks as "non-content" that occur after blocks that have been marked
- * {@link DefaultLabels#INDICATES_END_OF_TEXT}, and after any content block. This filter can be used
- * in conjunction with an upstream {@link TerminatingBlocksFinder}.
- *
- * @see TerminatingBlocksFinder
- */
-public final class IgnoreBlocksAfterContentFromEndFilter extends HeuristicFilterBase implements
- BoilerpipeFilter {
- public static final IgnoreBlocksAfterContentFromEndFilter INSTANCE =
- new IgnoreBlocksAfterContentFromEndFilter();
-
- private IgnoreBlocksAfterContentFromEndFilter() {
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- boolean changes = false;
-
- int words = 0;
-
- List<TextBlock> blocks = doc.getTextBlocks();
- if (!blocks.isEmpty()) {
- ListIterator<TextBlock> it = blocks.listIterator(blocks.size());
-
- TextBlock tb;
-
- while (it.hasPrevious()) {
- tb = it.previous();
- if (tb.hasLabel(DefaultLabels.INDICATES_END_OF_TEXT)) {
- tb.addLabel(DefaultLabels.STRICTLY_NOT_CONTENT);
- tb.removeLabel(DefaultLabels.MIGHT_BE_CONTENT);
- tb.setIsContent(false);
- changes = true;
- } else if (tb.isContent()) {
- words += tb.getNumWords();
- if (words > 200) {
- break;
- }
- }
-
- }
- }
-
- return changes;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/KeepLargestFulltextBlockFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/KeepLargestFulltextBlockFilter.java
deleted file mode 100644
index d01fc554b3..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/KeepLargestFulltextBlockFilter.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.english;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.filters.heuristics.KeepLargestBlockFilter;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.List;
-
-/**
- * Keeps the largest {@link TextBlock} only (by the number of words). In case of more than one block
- * with the same number of words, the first block is chosen. All discarded blocks are marked
- * "not content" and flagged as {@link DefaultLabels#MIGHT_BE_CONTENT}.
- *
- * As opposed to {@link KeepLargestBlockFilter}, the number of words are computed using
- * {@link HeuristicFilterBase#getNumFullTextWords(TextBlock)}, which only counts words that occur in
- * text elements with at least 9 words and are thus believed to be full text.
- *
- * NOTE: Without language-specific fine-tuning (i.e., running the default instance), this filter may
- * lead to suboptimal results. You better use {@link KeepLargestBlockFilter} instead, which works at
- * the level of number-of-words instead of text densities.
- */
-public final class KeepLargestFulltextBlockFilter extends HeuristicFilterBase implements
- BoilerpipeFilter {
- public static final KeepLargestFulltextBlockFilter INSTANCE =
- new KeepLargestFulltextBlockFilter();
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- if (textBlocks.size() < 2) {
- return false;
- }
-
- int max = -1;
- TextBlock largestBlock = null;
- for (TextBlock tb : textBlocks) {
- if (!tb.isContent()) {
- continue;
- }
- int numWords = getNumFullTextWords(tb);
- if (numWords > max) {
- largestBlock = tb;
- max = numWords;
- }
- }
-
- if (largestBlock == null) {
- return false;
- }
-
- for (TextBlock tb : textBlocks) {
- if (tb == largestBlock) {
- tb.setIsContent(true);
- } else {
- tb.setIsContent(false);
- tb.addLabel(DefaultLabels.MIGHT_BE_CONTENT);
- }
- }
-
- return true;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/MinFulltextWordsFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/MinFulltextWordsFilter.java
deleted file mode 100644
index b13cf71167..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/MinFulltextWordsFilter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.english;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * Keeps only those content blocks which contain at least k full-text words (measured by
- * {@link HeuristicFilterBase#getNumFullTextWords(TextBlock)}). k is 30 by default.
- */
-public final class MinFulltextWordsFilter extends HeuristicFilterBase implements BoilerpipeFilter {
- public static final MinFulltextWordsFilter DEFAULT_INSTANCE = new MinFulltextWordsFilter(30);
- private final int minWords;
-
- public static MinFulltextWordsFilter getDefaultInstance() {
- return DEFAULT_INSTANCE;
- }
-
- public MinFulltextWordsFilter(final int minWords) {
- this.minWords = minWords;
- }
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
-
- boolean changes = false;
-
- for (TextBlock tb : doc.getTextBlocks()) {
- if (!tb.isContent()) {
- continue;
- }
- if (getNumFullTextWords(tb) < minWords) {
- tb.setIsContent(false);
- changes = true;
- }
-
- }
-
- return changes;
-
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/NumWordsRulesClassifier.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/NumWordsRulesClassifier.java
deleted file mode 100644
index 7e9091bf4a..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/NumWordsRulesClassifier.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.english;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Classifies {@link TextBlock}s as content/not-content through rules that have been determined
- * using the C4.8 machine learning algorithm, as described in the paper
- * "Boilerplate Detection using Shallow Text Features" (WSDM 2010), particularly using number of
- * words per block and link density per block.
- */
-public class NumWordsRulesClassifier implements BoilerpipeFilter {
- public static final NumWordsRulesClassifier INSTANCE = new NumWordsRulesClassifier();
-
- /**
- * Returns the singleton instance for RulebasedBoilerpipeClassifier.
- */
- public static NumWordsRulesClassifier getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- boolean hasChanges = false;
-
- ListIterator<TextBlock> it = textBlocks.listIterator();
- if (!it.hasNext()) {
- return false;
- }
- TextBlock prevBlock = TextBlock.EMPTY_START;
- TextBlock currentBlock = it.next();
- TextBlock nextBlock = it.hasNext() ? it.next() : TextBlock.EMPTY_START;
-
- hasChanges = classify(prevBlock, currentBlock, nextBlock) | hasChanges;
-
- if (nextBlock != TextBlock.EMPTY_START) {
- while (it.hasNext()) {
- prevBlock = currentBlock;
- currentBlock = nextBlock;
- nextBlock = it.next();
- hasChanges = classify(prevBlock, currentBlock, nextBlock) | hasChanges;
- }
- prevBlock = currentBlock;
- currentBlock = nextBlock;
- nextBlock = TextBlock.EMPTY_START;
- hasChanges = classify(prevBlock, currentBlock, nextBlock) | hasChanges;
- }
-
- return hasChanges;
- }
-
- protected boolean classify(final TextBlock prev, final TextBlock curr, final TextBlock next) {
- final boolean isContent;
-
- if (curr.getLinkDensity() <= 0.333333) {
- if (prev.getLinkDensity() <= 0.555556) {
- if (curr.getNumWords() <= 16) {
- if (next.getNumWords() <= 15) {
- if (prev.getNumWords() <= 4) {
- isContent = false;
- } else {
- isContent = true;
- }
- } else {
- isContent = true;
- }
- } else {
- isContent = true;
- }
- } else {
- if (curr.getNumWords() <= 40) {
- if (next.getNumWords() <= 17) {
- isContent = false;
- } else {
- isContent = true;
- }
- } else {
- isContent = true;
- }
- }
- } else {
- isContent = false;
- }
-
- return curr.setIsContent(isContent);
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/TerminatingBlocksFinder.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/TerminatingBlocksFinder.java
deleted file mode 100644
index 081ece58fe..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/TerminatingBlocksFinder.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.english;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-/**
- * Finds blocks which are potentially indicating the end of an article text and marks them with
- * {@link DefaultLabels#INDICATES_END_OF_TEXT}. This can be used in conjunction with a downstream
- * {@link IgnoreBlocksAfterContentFilter}.
- *
- * @see IgnoreBlocksAfterContentFilter
- */
-public class TerminatingBlocksFinder implements BoilerpipeFilter {
- public static final TerminatingBlocksFinder INSTANCE = new TerminatingBlocksFinder();
-
- /**
- * Returns the singleton instance for TerminatingBlocksFinder.
- */
- public static TerminatingBlocksFinder getInstance() {
- return INSTANCE;
- }
-
- // public static long timeSpent = 0;
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- boolean changes = false;
-
- // long t = System.currentTimeMillis();
-
- for (TextBlock tb : doc.getTextBlocks()) {
- final int numWords = tb.getNumWords();
- if (numWords < 15) {
- final String text = tb.getText().trim();
- final int len = text.length();
- if (len >= 8) {
- final String textLC = text.toLowerCase();
- if (textLC.startsWith("comments")
- || startsWithNumber(textLC, len, " comments", " users responded in")
- || textLC.startsWith("© reuters") || textLC.startsWith("please rate this")
- || textLC.startsWith("post a comment") || textLC.contains("what you think...")
- || textLC.contains("add your comment") || textLC.contains("add comment")
- || textLC.contains("reader views") || textLC.contains("have your say")
- || textLC.contains("reader comments") || textLC.contains("rätta artikeln")
- || textLC.equals("thanks for your comments - this feedback is now closed")) {
- tb.addLabel(DefaultLabels.INDICATES_END_OF_TEXT);
- changes = true;
- }
- } else if (tb.getLinkDensity() == 1.0) {
- if (text.equals("Comment")) {
- tb.addLabel(DefaultLabels.INDICATES_END_OF_TEXT);
- }
- }
- }
- }
-
- // timeSpent += System.currentTimeMillis() - t;
-
- return changes;
- }
-
- /**
- * Checks whether the given text t starts with a sequence of digits, followed by one of the given
- * strings.
- *
- * @param t The text to examine
- * @param len The length of the text to examine
- * @param str Any strings that may follow the digits.
- * @return true if at least one combination matches
- */
- private static boolean startsWithNumber(final String t, final int len, final String... str) {
- int j = 0;
- while (j < len && isDigit(t.charAt(j))) {
- j++;
- }
- if (j != 0) {
- for (String s : str) {
- if (t.startsWith(s, j)) {
- return true;
- }
- }
- }
- return false;
- }
-
- private static boolean isDigit(final char c) {
- return c >= '0' && c <= '9';
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/package-info.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/package-info.java
deleted file mode 100644
index 9998b5283d..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/english/package-info.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * These BoilerpipeFilters have only been tested on English text.
- * <p>
- * That is, they will probably work with other Western languages, but maybe need some parameter tuning to perform well.
- */
-package com.kohlschutter.boilerpipe.filters.english;
-
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/AddPrecedingLabelsFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/AddPrecedingLabelsFilter.java
deleted file mode 100644
index 22d76970f1..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/AddPrecedingLabelsFilter.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Set;
-
-/**
- * Adds the labels of the preceding block to the current block, optionally adding a prefix.
- */
-public final class AddPrecedingLabelsFilter implements BoilerpipeFilter {
-
- public static final AddPrecedingLabelsFilter INSTANCE = new AddPrecedingLabelsFilter("");
- public static final AddPrecedingLabelsFilter INSTANCE_PRE = new AddPrecedingLabelsFilter("^");
-
- private final String labelPrefix;
-
- /**
- * Creates a new {@link AddPrecedingLabelsFilter} instance.
- *
- * @param maxBlocksDistance The maximum distance in blocks.
- * @param contentOnly
- */
- public AddPrecedingLabelsFilter(final String labelPrefix) {
- this.labelPrefix = labelPrefix;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- if (textBlocks.size() < 2) {
- return false;
- }
-
- boolean changes = false;
- int remaining = textBlocks.size();
-
- TextBlock blockBelow = null;
- TextBlock block;
- for (ListIterator<TextBlock> it = textBlocks.listIterator(textBlocks.size()); it.hasPrevious(); ) {
- if (--remaining <= 0) {
- break;
- }
- if (blockBelow == null) {
- blockBelow = it.previous();
- continue;
- }
- block = it.previous();
-
- Set<String> labels = block.getLabels();
- if (labels != null && !labels.isEmpty()) {
- for (String l : labels) {
- blockBelow.addLabel(labelPrefix + l);
- }
- changes = true;
- }
- blockBelow = block;
- }
-
- return changes;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ArticleMetadataFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ArticleMetadataFilter.java
deleted file mode 100644
index 0a94abf333..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ArticleMetadataFilter.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.regex.Pattern;
-
-/**
- * Tries to find TextBlocks that comprise of "article metadata".
- */
-public class ArticleMetadataFilter implements BoilerpipeFilter {
- private static final Pattern[] PATTERNS_SHORT =
- new Pattern[]{
- Pattern
- .compile(
- "^[0-9 \\,\\./]*\\b(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec|January|February|March|April|May|June|July|August|September|October|November|December)?\\b[0-9 \\,\\:apm\\./]*([CPSDMGET]{2,3})?$"),
- Pattern.compile("^[Bb]y ")};
-
- public static final ArticleMetadataFilter INSTANCE = new ArticleMetadataFilter();
-
- private ArticleMetadataFilter() {
- }
-
- @Override
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- boolean changed = false;
- for (TextBlock tb : doc.getTextBlocks()) {
- if (tb.getNumWords() > 10) {
- continue;
- }
- final String text = tb.getText();
- for (Pattern p : PATTERNS_SHORT) {
- if (p.matcher(text).find()) {
- changed = true;
- tb.setIsContent(true);
- tb.addLabel(DefaultLabels.ARTICLE_METADATA);
- }
- }
- }
- return changed;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/BlockProximityFusion.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/BlockProximityFusion.java
deleted file mode 100644
index b2538d8c48..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/BlockProximityFusion.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Fuses adjacent blocks if their distance (in blocks) does not exceed a certain limit. This
- * probably makes sense only in cases where an upstream filter already has removed some blocks.
- */
-public final class BlockProximityFusion implements BoilerpipeFilter {
-
- private final int maxBlocksDistance;
-
- public static final BlockProximityFusion MAX_DISTANCE_1 = new BlockProximityFusion(1, false,
- false);
- public static final BlockProximityFusion MAX_DISTANCE_1_SAME_TAGLEVEL = new BlockProximityFusion(
- 1, false, true);
- public static final BlockProximityFusion MAX_DISTANCE_1_CONTENT_ONLY = new BlockProximityFusion(
- 1, true, false);
- public static final BlockProximityFusion MAX_DISTANCE_1_CONTENT_ONLY_SAME_TAGLEVEL =
- new BlockProximityFusion(1, true, true);
-
- private final boolean contentOnly;
-
- private final boolean sameTagLevelOnly;
-
- /**
- * Creates a new {@link BlockProximityFusion} instance.
- *
- * @param maxBlocksDistance The maximum distance in blocks.
- * @param contentOnly
- */
- public BlockProximityFusion(final int maxBlocksDistance, final boolean contentOnly,
- final boolean sameTagLevelOnly) {
- this.maxBlocksDistance = maxBlocksDistance;
- this.contentOnly = contentOnly;
- this.sameTagLevelOnly = sameTagLevelOnly;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- if (textBlocks.size() < 2) {
- return false;
- }
-
- boolean changes = false;
- TextBlock prevBlock;
-
- int offset;
- if (contentOnly) {
- prevBlock = null;
- offset = 0;
- for (TextBlock tb : textBlocks) {
- offset++;
- if (tb.isContent()) {
- prevBlock = tb;
- break;
- }
- }
- if (prevBlock == null) {
- return false;
- }
- } else {
- prevBlock = textBlocks.get(0);
- offset = 1;
- }
-
- for (Iterator<TextBlock> it = textBlocks.listIterator(offset); it.hasNext(); ) {
- TextBlock block = it.next();
- if (!block.isContent()) {
- prevBlock = block;
- continue;
- }
- int diffBlocks = block.getOffsetBlocksStart() - prevBlock.getOffsetBlocksEnd() - 1;
- if (diffBlocks <= maxBlocksDistance) {
- boolean ok = true;
- if (contentOnly) {
- if (!prevBlock.isContent() || !block.isContent()) {
- ok = false;
- }
- }
- if (ok && sameTagLevelOnly && prevBlock.getTagLevel() != block.getTagLevel()) {
- ok = false;
- }
- if (ok) {
- prevBlock.mergeNext(block);
- it.remove();
- changes = true;
- } else {
- prevBlock = block;
- }
- } else {
- prevBlock = block;
- }
- }
-
- return changes;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ContentFusion.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ContentFusion.java
deleted file mode 100644
index 1e6dfbfed8..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ContentFusion.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Merges two blocks using some heuristics.
- */
-public final class ContentFusion implements BoilerpipeFilter {
-
- public static final ContentFusion INSTANCE = new ContentFusion();
-
- /**
- * Creates a new {@link ContentFusion} instance.
- *
- */
- public ContentFusion() {
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- if (textBlocks.size() < 2) {
- return false;
- }
-
- TextBlock prevBlock = textBlocks.get(0);
-
- boolean changes = false;
- do {
- changes = false;
- for (ListIterator<TextBlock> it = textBlocks.listIterator(1); it.hasNext(); ) {
- TextBlock block = it.next();
-
- if (prevBlock.isContent() && block.getLinkDensity() < 0.56
- && !block.hasLabel(DefaultLabels.STRICTLY_NOT_CONTENT)) {
-
- prevBlock.mergeNext(block);
- it.remove();
- changes = true;
- } else {
- prevBlock = block;
- }
- }
- } while (changes);
-
- return true;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/DocumentTitleMatchClassifier.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/DocumentTitleMatchClassifier.java
deleted file mode 100644
index 3bcf9f6cfa..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/DocumentTitleMatchClassifier.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-/**
- * Marks {@link TextBlock}s which contain parts of the HTML <code><TITLE></code> tag, using
- * some heuristics which are quite specific to the news domain.
- */
-public final class DocumentTitleMatchClassifier implements BoilerpipeFilter {
-
- private final Set<String> potentialTitles;
-
- public DocumentTitleMatchClassifier(String title) {
- if (title == null) {
- this.potentialTitles = null;
- } else {
-
- title = title.replace('\u00a0', ' ');
- title = title.replace("'", "");
-
- title = title.trim().toLowerCase();
-
- if (title.length() == 0) {
- this.potentialTitles = null;
- } else {
- this.potentialTitles = new HashSet<String>();
-
- potentialTitles.add(title);
-
- String p;
-
- p = getLongestPart(title, "[ ]*[\\|»|-][ ]*");
- if (p != null) {
- potentialTitles.add(p);
- }
- p = getLongestPart(title, "[ ]*[\\|»|:][ ]*");
- if (p != null) {
- potentialTitles.add(p);
- }
- p = getLongestPart(title, "[ ]*[\\|»|:\\(\\)][ ]*");
- if (p != null) {
- potentialTitles.add(p);
- }
- p = getLongestPart(title, "[ ]*[\\|»|:\\(\\)\\-][ ]*");
- if (p != null) {
- potentialTitles.add(p);
- }
- p = getLongestPart(title, "[ ]*[\\|»|,|:\\(\\)\\-][ ]*");
- if (p != null) {
- potentialTitles.add(p);
- }
- p = getLongestPart(title, "[ ]*[\\|»|,|:\\(\\)\\-\u00a0][ ]*");
- if (p != null) {
- potentialTitles.add(p);
- }
-
- addPotentialTitles(potentialTitles, title, "[ ]+[\\|][ ]+", 4);
- addPotentialTitles(potentialTitles, title, "[ ]+[\\-][ ]+", 4);
-
- potentialTitles.add(title.replaceFirst(" - [^\\-]+$", ""));
- potentialTitles.add(title.replaceFirst("^[^\\-]+ - ", ""));
- }
- }
- }
-
- public Set<String> getPotentialTitles() {
- return potentialTitles;
- }
-
- private void addPotentialTitles(final Set<String> potentialTitles, final String title,
- final String pattern, final int minWords) {
- String[] parts = title.split(pattern);
- if (parts.length == 1) {
- return;
- }
- for (int i = 0; i < parts.length; i++) {
- String p = parts[i];
- if (p.contains(".com")) {
- continue;
- }
- final int numWords = p.split("[\b ]+").length;
- if (numWords >= minWords) {
- potentialTitles.add(p);
- }
- }
- }
-
- private String getLongestPart(final String title, final String pattern) {
- String[] parts = title.split(pattern);
- if (parts.length == 1) {
- return null;
- }
- int longestNumWords = 0;
- String longestPart = "";
- for (int i = 0; i < parts.length; i++) {
- String p = parts[i];
- if (p.contains(".com")) {
- continue;
- }
- final int numWords = p.split("[\b ]+").length;
- if (numWords > longestNumWords || p.length() > longestPart.length()) {
- longestNumWords = numWords;
- longestPart = p;
- }
- }
- if (longestPart.length() == 0) {
- return null;
- } else {
- return longestPart.trim();
- }
- }
-
- private static final Pattern PAT_REMOVE_CHARACTERS = Pattern.compile("[\\?\\!\\.\\-\\:]+");
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- if (potentialTitles == null) {
- return false;
- }
- boolean changes = false;
-
- for (final TextBlock tb : doc.getTextBlocks()) {
- String text = tb.getText();
-
- text = text.replace('\u00a0', ' ');
- text = text.replace("'", "");
-
- text = text.trim().toLowerCase();
-
- if (potentialTitles.contains(text)) {
- tb.addLabel(DefaultLabels.TITLE);
- changes = true;
- break;
- }
-
- text = PAT_REMOVE_CHARACTERS.matcher(text).replaceAll("").trim();
- if (potentialTitles.contains(text)) {
- tb.addLabel(DefaultLabels.TITLE);
- changes = true;
- break;
- }
- }
- return changes;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ExpandTitleToContentFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ExpandTitleToContentFilter.java
deleted file mode 100644
index 423597e7e1..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ExpandTitleToContentFilter.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-/**
- * Marks all {@link TextBlock}s "content" which are between the headline and the part that has
- * already been marked content, if they are marked {@link DefaultLabels#MIGHT_BE_CONTENT}.
- *
- * This filter is quite specific to the news domain.
- */
-public final class ExpandTitleToContentFilter implements BoilerpipeFilter {
- public static final ExpandTitleToContentFilter INSTANCE = new ExpandTitleToContentFilter();
-
- /**
- * Returns the singleton instance for ExpandTitleToContentFilter.
- */
- public static ExpandTitleToContentFilter getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- int i = 0;
- int title = -1;
- int contentStart = -1;
- for (TextBlock tb : doc.getTextBlocks()) {
- if (contentStart == -1 && tb.hasLabel(DefaultLabels.TITLE)) {
- title = i;
- contentStart = -1;
- }
- if (contentStart == -1 && tb.isContent()) {
- contentStart = i;
- }
-
- i++;
- }
-
- if (contentStart <= title || title == -1) {
- return false;
- }
- boolean changes = false;
- for (TextBlock tb : doc.getTextBlocks().subList(title, contentStart)) {
- if (tb.hasLabel(DefaultLabels.MIGHT_BE_CONTENT)) {
- changes = tb.setIsContent(true) | changes;
- }
- }
- return changes;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/KeepLargestBlockFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/KeepLargestBlockFilter.java
deleted file mode 100644
index d04703394a..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/KeepLargestBlockFilter.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Keeps the largest {@link TextBlock} only (by the number of words). In case of more than one block
- * with the same number of words, the first block is chosen. All discarded blocks are marked
- * "not content" and flagged as {@link DefaultLabels#MIGHT_BE_CONTENT}.
- *
- * Note that, by default, only TextBlocks marked as "content" are taken into consideration.
- */
-public final class KeepLargestBlockFilter implements BoilerpipeFilter {
- public static final KeepLargestBlockFilter INSTANCE = new KeepLargestBlockFilter(false, 0);
- public static final KeepLargestBlockFilter INSTANCE_EXPAND_TO_SAME_TAGLEVEL =
- new KeepLargestBlockFilter(true, 0);
- public static final KeepLargestBlockFilter INSTANCE_EXPAND_TO_SAME_TAGLEVEL_MIN_WORDS =
- new KeepLargestBlockFilter(true, 150);
- private final boolean expandToSameLevelText;
- private final int minWords;
-
- public KeepLargestBlockFilter(boolean expandToSameLevelText, final int minWords) {
- this.expandToSameLevelText = expandToSameLevelText;
- this.minWords = minWords;
- }
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- if (textBlocks.size() < 2) {
- return false;
- }
-
- int maxNumWords = -1;
- TextBlock largestBlock = null;
-
- int level = -1;
-
- int i = 0;
- int n = -1;
- for (TextBlock tb : textBlocks) {
- if (tb.isContent()) {
- final int nw = tb.getNumWords();
-
- if (nw > maxNumWords) {
- largestBlock = tb;
- maxNumWords = nw;
-
- n = i;
-
- if (expandToSameLevelText) {
- level = tb.getTagLevel();
- }
- }
- }
- i++;
- }
- for (TextBlock tb : textBlocks) {
- if (tb == largestBlock) {
- tb.setIsContent(true);
- tb.addLabel(DefaultLabels.VERY_LIKELY_CONTENT);
- } else {
- tb.setIsContent(false);
- tb.addLabel(DefaultLabels.MIGHT_BE_CONTENT);
- }
- }
- if (expandToSameLevelText && n != -1) {
-
- for (ListIterator<TextBlock> it = textBlocks.listIterator(n); it.hasPrevious(); ) {
- TextBlock tb = it.previous();
- final int tl = tb.getTagLevel();
- if (tl < level) {
- break;
- } else if (tl == level) {
- if (tb.getNumWords() >= minWords) {
- tb.setIsContent(true);
- }
- }
- }
- for (ListIterator<TextBlock> it = textBlocks.listIterator(n); it.hasNext(); ) {
- TextBlock tb = it.next();
- final int tl = tb.getTagLevel();
- if (tl < level) {
- break;
- } else if (tl == level) {
- if (tb.getNumWords() >= minWords) {
- tb.setIsContent(true);
- }
- }
- }
- }
-
- return true;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/LabelFusion.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/LabelFusion.java
deleted file mode 100644
index 07d1c9f636..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/LabelFusion.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Fuses adjacent blocks if their labels are equal.
- */
-public final class LabelFusion implements BoilerpipeFilter {
-
- public static final LabelFusion INSTANCE = new LabelFusion();
-
- /**
- * Creates a new {@link LabelFusion} instance.
- */
- private LabelFusion() {
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- if (textBlocks.size() < 2) {
- return false;
- }
-
- boolean changes = false;
- TextBlock prevBlock = textBlocks.get(0);
- int offset = 1;
-
- for (Iterator<TextBlock> it = textBlocks.listIterator(offset); it.hasNext(); ) {
- TextBlock block = it.next();
-
- if (equalLabels(prevBlock.getLabels(), block.getLabels())) {
- prevBlock.mergeNext(block);
- it.remove();
- changes = true;
- } else {
- prevBlock = block;
- }
- }
-
- return changes;
- }
-
- private boolean equalLabels(Set<String> labels, Set<String> labels2) {
- if (labels == null || labels2 == null) {
- return false;
- }
- return markupLabelsOnly(labels).equals(markupLabelsOnly(labels2));
- }
-
- private Set<String> markupLabelsOnly(final Set<String> set1) {
- Set<String> set = new HashSet<String>(set1);
- for (Iterator<String> it = set.iterator(); it.hasNext(); ) {
- final String label = it.next();
- if (!label.startsWith(DefaultLabels.MARKUP_PREFIX)) {
- it.remove();
- }
- }
- return set;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/LargeBlockSameTagLevelToContentFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/LargeBlockSameTagLevelToContentFilter.java
deleted file mode 100644
index 1fc5df4d4a..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/LargeBlockSameTagLevelToContentFilter.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-/**
- * Marks all blocks as content that:
- * <ol>
- * <li>are on the same tag-level as very likely main content (usually the level of the largest
- * block)</li>
- * <li>have a significant number of words, currently: at least 100</li>
- * </ol>
- */
-public final class LargeBlockSameTagLevelToContentFilter implements BoilerpipeFilter {
- public static final LargeBlockSameTagLevelToContentFilter INSTANCE =
- new LargeBlockSameTagLevelToContentFilter();
-
- private LargeBlockSameTagLevelToContentFilter() {
- }
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
-
- boolean changes = false;
-
- int tagLevel = -1;
- for (TextBlock tb : doc.getTextBlocks()) {
- if (tb.isContent() && tb.hasLabel(DefaultLabels.VERY_LIKELY_CONTENT)) {
- tagLevel = tb.getTagLevel();
- break;
- }
- }
-
- if (tagLevel == -1) {
- return false;
- }
-
- for (TextBlock tb : doc.getTextBlocks()) {
- if (!tb.isContent()) {
-
- if (tb.getNumWords() >= 100 && tb.getTagLevel() == tagLevel) {
- tb.setIsContent(true);
- changes = true;
- }
- }
- }
-
- return changes;
-
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ListAtEndFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ListAtEndFilter.java
deleted file mode 100644
index 1b316fb51e..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/ListAtEndFilter.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-/**
- * Marks nested list-item blocks after the end of the main content.
- */
-public final class ListAtEndFilter implements BoilerpipeFilter {
- public static final ListAtEndFilter INSTANCE = new ListAtEndFilter();
-
- private ListAtEndFilter() {
- }
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
-
- boolean changes = false;
-
- int tagLevel = Integer.MAX_VALUE;
- for (TextBlock tb : doc.getTextBlocks()) {
- if (tb.isContent() && tb.hasLabel(DefaultLabels.VERY_LIKELY_CONTENT)) {
- tagLevel = tb.getTagLevel();
- } else {
- if (tb.getTagLevel() > tagLevel && tb.hasLabel(DefaultLabels.MIGHT_BE_CONTENT)
- && tb.hasLabel(DefaultLabels.LI) && tb.getLinkDensity() == 0) {
- tb.setIsContent(true);
- changes = true;
- } else {
- tagLevel = Integer.MAX_VALUE;
- }
- }
- }
-
- return changes;
-
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/SimpleBlockFusionProcessor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/SimpleBlockFusionProcessor.java
deleted file mode 100644
index c7109e791e..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/SimpleBlockFusionProcessor.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Merges two subsequent blocks if their text densities are equal.
- */
-public class SimpleBlockFusionProcessor implements BoilerpipeFilter {
- public static final SimpleBlockFusionProcessor INSTANCE = new SimpleBlockFusionProcessor();
-
- /**
- * Returns the singleton instance for BlockFusionProcessor.
- */
- public static SimpleBlockFusionProcessor getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- boolean changes = false;
-
- if (textBlocks.size() < 2) {
- return false;
- }
-
- TextBlock b1 = textBlocks.get(0);
- for (Iterator<TextBlock> it = textBlocks.listIterator(1); it.hasNext(); ) {
- TextBlock b2 = it.next();
-
- final boolean similar = (b1.getTextDensity() == b2.getTextDensity());
-
- if (similar) {
- b1.mergeNext(b2);
- it.remove();
- changes = true;
- } else {
- b1 = b2;
- }
- }
-
- return changes;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/TrailingHeadlineToBoilerplateFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/TrailingHeadlineToBoilerplateFilter.java
deleted file mode 100644
index 91f4610e75..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/TrailingHeadlineToBoilerplateFilter.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Marks trailing headlines ({@link TextBlock}s that have the label {@link DefaultLabels#HEADING})
- * as boilerplate. Trailing means they are marked content and are below any other content block.
- */
-public final class TrailingHeadlineToBoilerplateFilter implements BoilerpipeFilter {
- public static final TrailingHeadlineToBoilerplateFilter INSTANCE =
- new TrailingHeadlineToBoilerplateFilter();
-
- /**
- * Returns the singleton instance for ExpandTitleToContentFilter.
- */
- public static TrailingHeadlineToBoilerplateFilter getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- boolean changes = false;
-
- List<TextBlock> list = doc.getTextBlocks();
-
- for (ListIterator<TextBlock> it = list.listIterator(list.size()); it.hasPrevious(); ) {
- TextBlock tb = it.previous();
- if (tb.isContent()) {
- if (tb.hasLabel(DefaultLabels.HEADING)) {
- tb.setIsContent(false);
- changes = true;
- } else {
- break;
- }
- }
- }
-
- return changes;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/package-info.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/package-info.java
deleted file mode 100644
index 969370248c..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/heuristics/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * These BoilerpipeFilters are pure heuristics.
- */
-package com.kohlschutter.boilerpipe.filters.heuristics;
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/BoilerplateBlockFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/BoilerplateBlockFilter.java
deleted file mode 100644
index 85e9ccc388..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/BoilerplateBlockFilter.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Removes {@link TextBlock}s which have explicitly been marked as "not content".
- */
-public final class BoilerplateBlockFilter implements BoilerpipeFilter {
- public static final BoilerplateBlockFilter INSTANCE = new BoilerplateBlockFilter(null);
- public static final BoilerplateBlockFilter INSTANCE_KEEP_TITLE = new BoilerplateBlockFilter(
- DefaultLabels.TITLE);
- private final String labelToKeep;
-
- /**
- * Returns the singleton instance for BoilerplateBlockFilter.
- */
- public static BoilerplateBlockFilter getInstance() {
- return INSTANCE;
- }
-
- public BoilerplateBlockFilter(final String labelToKeep) {
- this.labelToKeep = labelToKeep;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- List<TextBlock> textBlocks = doc.getTextBlocks();
- boolean hasChanges = false;
-
- for (Iterator<TextBlock> it = textBlocks.iterator(); it.hasNext(); ) {
- TextBlock tb = it.next();
- if (!tb.isContent() && (labelToKeep == null || !tb.hasLabel(DefaultLabels.TITLE))) {
- it.remove();
- hasChanges = true;
- }
- }
-
- return hasChanges;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/InvertedFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/InvertedFilter.java
deleted file mode 100644
index 260e90756f..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/InvertedFilter.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.util.List;
-
-/**
- * Reverts the "isContent" flag for all {@link TextBlock}s
- */
-public final class InvertedFilter implements BoilerpipeFilter {
- public static final InvertedFilter INSTANCE = new InvertedFilter();
-
- private InvertedFilter() {
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
-
- List<TextBlock> tbs = doc.getTextBlocks();
- if (tbs.isEmpty()) {
- return false;
- }
- for (TextBlock tb : tbs) {
- tb.setIsContent(!tb.isContent());
- }
-
- return true;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/LabelToBoilerplateFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/LabelToBoilerplateFilter.java
deleted file mode 100644
index 9dad1856c4..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/LabelToBoilerplateFilter.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-
-/**
- * Marks all blocks that contain a given label as "boilerplate".
- */
-public final class LabelToBoilerplateFilter implements BoilerpipeFilter {
- public static final LabelToBoilerplateFilter INSTANCE_STRICTLY_NOT_CONTENT =
- new LabelToBoilerplateFilter(DefaultLabels.STRICTLY_NOT_CONTENT);
-
- private String[] labels;
-
- public LabelToBoilerplateFilter(final String... label) {
- this.labels = label;
- }
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
-
- boolean changes = false;
-
- BLOCK_LOOP:
- for (TextBlock tb : doc.getTextBlocks()) {
- if (tb.isContent()) {
- for (String label : labels) {
- if (tb.hasLabel(label)) {
- tb.setIsContent(false);
- changes = true;
- continue BLOCK_LOOP;
- }
- }
- }
- }
-
- return changes;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/LabelToContentFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/LabelToContentFilter.java
deleted file mode 100644
index 31bdbe9dc5..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/LabelToContentFilter.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * Marks all blocks that contain a given label as "content".
- */
-public final class LabelToContentFilter implements BoilerpipeFilter {
- private String[] labels;
-
- public LabelToContentFilter(final String... label) {
- this.labels = label;
- }
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
-
- boolean changes = false;
-
- BLOCK_LOOP:
- for (TextBlock tb : doc.getTextBlocks()) {
- if (!tb.isContent()) {
- for (String label : labels) {
- if (tb.hasLabel(label)) {
- tb.setIsContent(true);
- changes = true;
- continue BLOCK_LOOP;
- }
- }
- }
- }
-
- return changes;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MarkEverythingBoilerplateFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MarkEverythingBoilerplateFilter.java
deleted file mode 100644
index 67334bdbe0..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MarkEverythingBoilerplateFilter.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * Marks all blocks as boilerplate.
- */
-public final class MarkEverythingBoilerplateFilter implements BoilerpipeFilter {
- public static final MarkEverythingBoilerplateFilter INSTANCE =
- new MarkEverythingBoilerplateFilter();
-
- private MarkEverythingBoilerplateFilter() {
- }
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
-
- boolean changes = false;
-
- for (TextBlock tb : doc.getTextBlocks()) {
- if (tb.isContent()) {
- tb.setIsContent(false);
- changes = true;
- }
- }
-
- return changes;
-
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MarkEverythingContentFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MarkEverythingContentFilter.java
deleted file mode 100644
index 70595a1183..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MarkEverythingContentFilter.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * Marks all blocks as content.
- */
-public final class MarkEverythingContentFilter implements BoilerpipeFilter {
- public static final MarkEverythingContentFilter INSTANCE = new MarkEverythingContentFilter();
-
- private MarkEverythingContentFilter() {
- }
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
-
- boolean changes = false;
-
- for (TextBlock tb : doc.getTextBlocks()) {
- if (!tb.isContent()) {
- tb.setIsContent(true);
- changes = true;
- }
- }
-
- return changes;
-
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MinClauseWordsFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MinClauseWordsFilter.java
deleted file mode 100644
index e2e607400a..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MinClauseWordsFilter.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Keeps only blocks that have at least one segment fragment ("clause") with at least <em>k</em>
- * words (default: 5).
- *
- * NOTE: You might consider using the {@link SplitParagraphBlocksFilter} upstream.
- *
- * @see SplitParagraphBlocksFilter
- */
-public final class MinClauseWordsFilter implements BoilerpipeFilter {
- public static final MinClauseWordsFilter INSTANCE = new MinClauseWordsFilter(5, false);
- private int minWords;
- private final boolean acceptClausesWithoutDelimiter;
-
- public MinClauseWordsFilter(final int minWords) {
- this(minWords, false);
- }
-
- public MinClauseWordsFilter(final int minWords, final boolean acceptClausesWithoutDelimiter) {
- this.minWords = minWords;
- this.acceptClausesWithoutDelimiter = acceptClausesWithoutDelimiter;
- }
-
- private final Pattern PAT_CLAUSE_DELIMITER = Pattern
- .compile("[\\p{L}\\d][\\,\\.\\:\\;\\!\\?]+([ \\n\\r]+|$)");
- private final Pattern PAT_WHITESPACE = Pattern.compile("[ \\n\\r]+");
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
-
- boolean changes = false;
- for (TextBlock tb : doc.getTextBlocks()) {
- if (!tb.isContent()) {
- continue;
- }
- final String text = tb.getText();
-
- Matcher m = PAT_CLAUSE_DELIMITER.matcher(text);
- boolean found = m.find();
- int start = 0;
- int end;
- boolean hasClause = false;
- while (found) {
- end = m.start() + 1;
- hasClause = isClause(text.subSequence(start, end));
- start = m.end();
-
- if (hasClause) {
- break;
- }
- found = m.find();
- }
- end = text.length();
-
- // since clauses should *always end* with a delimiter, we normally
- // don't consider text without one
- if (acceptClausesWithoutDelimiter) {
- hasClause |= isClause(text.subSequence(start, end));
- }
-
- if (!hasClause) {
- tb.setIsContent(false);
- changes = true;
- // System.err.println("IS NOT CONTENT: " + text);
- }
- }
-
- return changes;
-
- }
-
- private boolean isClause(final CharSequence text) {
- Matcher m = PAT_WHITESPACE.matcher(text);
- int n = 1;
- while (m.find()) {
- n++;
- if (n >= minWords) {
- return true;
- }
- }
- return n >= minWords;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MinWordsFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MinWordsFilter.java
deleted file mode 100644
index 372b94fbba..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/MinWordsFilter.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-/**
- * Keeps only those content blocks which contain at least <em>k</em> words.
- */
-public final class MinWordsFilter implements BoilerpipeFilter {
- private final int minWords;
-
- public MinWordsFilter(final int minWords) {
- this.minWords = minWords;
- }
-
- public boolean process(final TextDocument doc) throws BoilerpipeProcessingException {
-
- boolean changes = false;
-
- for (TextBlock tb : doc.getTextBlocks()) {
- if (!tb.isContent()) {
- continue;
- }
- if (tb.getNumWords() < minWords) {
- tb.setIsContent(false);
- changes = true;
- }
-
- }
-
- return changes;
-
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/SplitParagraphBlocksFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/SplitParagraphBlocksFilter.java
deleted file mode 100644
index def7fc9210..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/SplitParagraphBlocksFilter.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Splits TextBlocks at paragraph boundaries.
- *
- * NOTE: This is not fully supported (i.e., it will break highlighting support via
- * #getContainedTextElements()), but this one probably is necessary for some other filters.
- *
- * @see MinClauseWordsFilter
- */
-public final class SplitParagraphBlocksFilter implements BoilerpipeFilter {
- public static final SplitParagraphBlocksFilter INSTANCE = new SplitParagraphBlocksFilter();
-
- /**
- * Returns the singleton instance for TerminatingBlocksFinder.
- */
- public static SplitParagraphBlocksFilter getInstance() {
- return INSTANCE;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
- boolean changes = false;
-
- final List<TextBlock> blocks = doc.getTextBlocks();
- final List<TextBlock> blocksNew = new ArrayList<TextBlock>();
-
- for (TextBlock tb : blocks) {
- final String text = tb.getText();
- final String[] paragraphs = text.split("[\n\r]+");
- if (paragraphs.length < 2) {
- blocksNew.add(tb);
- continue;
- }
- final boolean isContent = tb.isContent();
- final Set<String> labels = tb.getLabels();
- for (String p : paragraphs) {
- final TextBlock tbP = new TextBlock(p);
- tbP.setIsContent(isContent);
- tbP.addLabels(labels);
- blocksNew.add(tbP);
- changes = true;
- }
- }
-
- if (changes) {
- blocks.clear();
- blocks.addAll(blocksNew);
- }
-
- return changes;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/SurroundingToContentFilter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/SurroundingToContentFilter.java
deleted file mode 100644
index 2e827d60da..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/SurroundingToContentFilter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
-import com.kohlschutter.boilerpipe.BoilerpipeFilter;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.conditions.TextBlockCondition;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Marks blocks as "content" if their preceding and following blocks are both already marked
- * "content", and the given {@link TextBlockCondition} is met.
- */
-public class SurroundingToContentFilter implements BoilerpipeFilter {
- public static final SurroundingToContentFilter INSTANCE_TEXT = new SurroundingToContentFilter(
- new TextBlockCondition() {
-
- @Override
- public boolean meetsCondition(TextBlock tb) {
- return tb.getLinkDensity() == 0 && tb.getNumWords() > 6;
- }
- });
-
- private final TextBlockCondition cond;
-
- public SurroundingToContentFilter(final TextBlockCondition cond) {
- this.cond = cond;
- }
-
- public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
-
- List<TextBlock> tbs = doc.getTextBlocks();
- if (tbs.size() < 3) {
- return false;
- }
-
- TextBlock a = tbs.get(0);
- TextBlock b = tbs.get(1);
- TextBlock c;
- boolean hasChanges = false;
- for (Iterator<TextBlock> it = tbs.listIterator(2); it.hasNext(); ) {
- c = it.next();
- if (!b.isContent() && a.isContent() && c.isContent() && cond.meetsCondition(b)) {
- b.setIsContent(true);
- hasChanges = true;
- }
-
- a = c;
- if (!it.hasNext()) {
- break;
- }
- b = it.next();
- }
-
- return hasChanges;
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/package-info.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/package-info.java
deleted file mode 100644
index 1730a33404..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/filters/simple/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * These BoilerpipeFilters are straight-forward and probably not really specific to English.
- */
-package com.kohlschutter.boilerpipe.filters.simple;
-
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/labels/ConditionalLabelAction.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/labels/ConditionalLabelAction.java
deleted file mode 100644
index e9926f1a8e..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/labels/ConditionalLabelAction.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.labels;
-
-import com.kohlschutter.boilerpipe.conditions.TextBlockCondition;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-
-/**
- * Adds labels to a {@link TextBlock} if the given criteria are met.
- */
-public final class ConditionalLabelAction extends LabelAction {
-
- private final TextBlockCondition condition;
-
- public ConditionalLabelAction(TextBlockCondition condition, String... labels) {
- super(labels);
- this.condition = condition;
- }
-
- public void addTo(final TextBlock tb) {
- if (condition.meetsCondition(tb)) {
- addLabelsTo(tb);
- }
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/labels/DefaultLabels.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/labels/DefaultLabels.java
deleted file mode 100644
index 60803d573d..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/labels/DefaultLabels.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.labels;
-
-import com.kohlschutter.boilerpipe.document.TextBlock;
-
-/**
- * Some pre-defined labels which can be used in conjunction with {@link TextBlock#addLabel(String)}
- * and {@link TextBlock#hasLabel(String)}.
- */
-public final class DefaultLabels {
- public static final String TITLE = "de.l3s.boilerpipe/TITLE";
- public static final String ARTICLE_METADATA = "de.l3s.boilerpipe/ARTICLE_METADATA";
- public static final String INDICATES_END_OF_TEXT = "de.l3s.boilerpipe/INDICATES_END_OF_TEXT";
- public static final String MIGHT_BE_CONTENT = "de.l3s.boilerpipe/MIGHT_BE_CONTENT";
- public static final String VERY_LIKELY_CONTENT = "de.l3s.boilerpipe/VERY_LIKELY_CONTENT";
- public static final String STRICTLY_NOT_CONTENT = "de.l3s.boilerpipe/STRICTLY_NOT_CONTENT";
- public static final String HR = "de.l3s.boilerpipe/HR";
- public static final String LI = "de.l3s.boilerpipe/LI";
-
- public static final String HEADING = "de.l3s.boilerpipe/HEADING";
- public static final String H1 = "de.l3s.boilerpipe/H1";
- public static final String H2 = "de.l3s.boilerpipe/H2";
- public static final String H3 = "de.l3s.boilerpipe/H3";
-
- public static final String MARKUP_PREFIX = "<";
-
- private DefaultLabels() {
- // not to be instantiated
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/labels/LabelAction.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/labels/LabelAction.java
deleted file mode 100644
index da9dd0e402..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/labels/LabelAction.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.labels;
-
-import com.kohlschutter.boilerpipe.document.TextBlock;
-
-import java.util.Arrays;
-
-/**
- * Helps adding labels to {@link TextBlock}s.
- *
- * @see ConditionalLabelAction
- */
-public class LabelAction {
- protected final String[] labels;
-
- public LabelAction(String... labels) {
- this.labels = labels;
- }
-
- public void addTo(final TextBlock tb) {
- addLabelsTo(tb);
- }
-
- protected final void addLabelsTo(final TextBlock tb) {
- tb.addLabels(labels);
- }
-
- public String toString() {
- return super.toString() + "{" + Arrays.asList(labels) + "}";
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/package-info.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/package-info.java
deleted file mode 100644
index 91422d676e..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * The Boilerpipe top-level package.
- */
-package com.kohlschutter.boilerpipe;
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/BoilerpipeHTMLContentHandler.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/BoilerpipeHTMLContentHandler.java
deleted file mode 100644
index cbd58fde0a..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/BoilerpipeHTMLContentHandler.java
+++ /dev/null
@@ -1,440 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import com.kohlschutter.boilerpipe.labels.LabelAction;
-import com.kohlschutter.boilerpipe.util.UnicodeTokenizer;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import org.xml.sax.Attributes;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.Locator;
-import org.xml.sax.SAXException;
-
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-/**
- * A simple SAX {@link ContentHandler}, used by {@link BoilerpipeSAXInput}. Can be used by different
- * parser implementations, e.g. NekoHTML and TagSoup.
- */
-public class BoilerpipeHTMLContentHandler implements ContentHandler {
-
- private final Map<String, TagAction> tagActions;
- private String title = null;
-
- static final String ANCHOR_TEXT_START = "$\ue00a<";
- static final String ANCHOR_TEXT_END = ">\ue00a$";
-
- StringBuilder tokenBuffer = new StringBuilder();
- StringBuilder textBuffer = new StringBuilder();
-
- int inBody = 0;
- int inAnchor = 0;
- int inIgnorableElement = 0;
-
- int tagLevel = 0;
- int blockTagLevel = -1;
-
- boolean sbLastWasWhitespace = false;
- private int textElementIdx = 0;
-
- private final List<TextBlock> textBlocks = new ArrayList<TextBlock>();
-
- private String lastStartTag = null;
- @SuppressWarnings("unused")
- private String lastEndTag = null;
- @SuppressWarnings("unused")
- private Event lastEvent = null;
-
- private int offsetBlocks = 0;
- private BitSet currentContainedTextElements = new BitSet();
-
- private boolean flush = false;
- boolean inAnchorText = false;
-
- LinkedList<LinkedList<LabelAction>> labelStacks = new LinkedList<LinkedList<LabelAction>>();
- LinkedList<Integer> fontSizeStack = new LinkedList<Integer>();
-
- /**
- * Recycles this instance.
- */
- public void recycle() {
- tokenBuffer.setLength(0);
- textBuffer.setLength(0);
-
- inBody = 0;
- inAnchor = 0;
- inIgnorableElement = 0;
- sbLastWasWhitespace = false;
- textElementIdx = 0;
-
- textBlocks.clear();
-
- lastStartTag = null;
- lastEndTag = null;
- lastEvent = null;
-
- offsetBlocks = 0;
- currentContainedTextElements.clear();
-
- flush = false;
- inAnchorText = false;
- }
-
- /**
- * Constructs a {@link BoilerpipeHTMLContentHandler} using the {@link DefaultTagActionMap}.
- */
- public BoilerpipeHTMLContentHandler() {
- this(DefaultTagActionMap.INSTANCE);
- }
-
- /**
- * Constructs a {@link BoilerpipeHTMLContentHandler} using the given {@link TagActionMap}.
- *
- * @param tagActions The {@link TagActionMap} to use, e.g. {@link DefaultTagActionMap}.
- */
- public BoilerpipeHTMLContentHandler(final TagActionMap tagActions) {
- this.tagActions = tagActions;
- }
-
- // @Override
- public void endDocument() throws SAXException {
- flushBlock();
- }
-
- // @Override
- public void endPrefixMapping(String prefix) throws SAXException {
- }
-
- // @Override
- public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException {
- if (!sbLastWasWhitespace) {
- textBuffer.append(' ');
- tokenBuffer.append(' ');
- }
- sbLastWasWhitespace = true;
- }
-
- // @Override
- public void processingInstruction(String target, String data) throws SAXException {
- }
-
- // @Override
- public void setDocumentLocator(Locator locator) {
- }
-
- // @Override
- public void skippedEntity(String name) throws SAXException {
- }
-
- // @Override
- public void startDocument() throws SAXException {
- }
-
- // @Override
- public void startPrefixMapping(String prefix, String uri) throws SAXException {
- }
-
- // @Override
- public void startElement(String uri, String localName, String qName, Attributes atts)
- throws SAXException {
- labelStacks.add(null);
-
- TagAction ta = tagActions.get(localName);
- if (ta != null) {
- if (ta.changesTagLevel()) {
- tagLevel++;
- }
- flush = ta.start(this, localName, qName, atts) | flush;
- } else {
- tagLevel++;
- flush = true;
- }
-
- lastEvent = Event.START_TAG;
- lastStartTag = localName;
- }
-
- // @Override
- public void endElement(String uri, String localName, String qName) throws SAXException {
- TagAction ta = tagActions.get(localName);
- if (ta != null) {
- flush = ta.end(this, localName, qName) | flush;
- } else {
- flush = true;
- }
-
- if (ta == null || ta.changesTagLevel()) {
- tagLevel--;
- }
-
- if (flush) {
- flushBlock();
- }
-
- lastEvent = Event.END_TAG;
- lastEndTag = localName;
-
- labelStacks.removeLast();
- }
-
- // @Override
- public void characters(char[] ch, int start, int length) throws SAXException {
- textElementIdx++;
-
- if (flush) {
- flushBlock();
- flush = false;
- }
-
- if (inIgnorableElement != 0) {
- return;
- }
-
- char c;
- boolean startWhitespace = false;
- boolean endWhitespace = false;
- if (length == 0) {
- return;
- }
-
- final int end = start + length;
- for (int i = start; i < end; i++) {
- if (Character.isWhitespace(ch[i])) {
- ch[i] = ' ';
- }
- }
- while (start < end) {
- c = ch[start];
- if (c == ' ') {
- startWhitespace = true;
- start++;
- length--;
- } else {
- break;
- }
- }
- while (length > 0) {
- c = ch[start + length - 1];
- if (c == ' ') {
- endWhitespace = true;
- length--;
- } else {
- break;
- }
- }
- if (length == 0) {
- if (startWhitespace || endWhitespace) {
- if (!sbLastWasWhitespace) {
- textBuffer.append(' ');
- tokenBuffer.append(' ');
- }
- sbLastWasWhitespace = true;
- } else {
- sbLastWasWhitespace = false;
- }
- lastEvent = Event.WHITESPACE;
- return;
- }
- if (startWhitespace) {
- if (!sbLastWasWhitespace) {
- textBuffer.append(' ');
- tokenBuffer.append(' ');
- }
- }
-
- if (blockTagLevel == -1) {
- blockTagLevel = tagLevel;
- }
-
- textBuffer.append(ch, start, length);
- tokenBuffer.append(ch, start, length);
- if (endWhitespace) {
- textBuffer.append(' ');
- tokenBuffer.append(' ');
- }
-
- sbLastWasWhitespace = endWhitespace;
- lastEvent = Event.CHARACTERS;
-
- currentContainedTextElements.set(textElementIdx);
- }
-
- List<TextBlock> getTextBlocks() {
- return textBlocks;
- }
-
- public void flushBlock() {
- if (inBody == 0) {
- if ("TITLE".equalsIgnoreCase(lastStartTag) && inBody == 0) {
- setTitle(tokenBuffer.toString().trim());
- }
- textBuffer.setLength(0);
- tokenBuffer.setLength(0);
- return;
- }
-
- final int length = tokenBuffer.length();
- switch (length) {
- case 0:
- return;
- case 1:
- if (sbLastWasWhitespace) {
- textBuffer.setLength(0);
- tokenBuffer.setLength(0);
- return;
- }
- }
- final String[] tokens = UnicodeTokenizer.tokenize(tokenBuffer);
-
- int numWords = 0;
- int numLinkedWords = 0;
- int numWrappedLines = 0;
- int currentLineLength = -1; // don't count the first space
- final int maxLineLength = 80;
- int numTokens = 0;
- int numWordsCurrentLine = 0;
-
- for (String token : tokens) {
- if (ANCHOR_TEXT_START.equals(token)) {
- inAnchorText = true;
- } else if (ANCHOR_TEXT_END.equals(token)) {
- inAnchorText = false;
- } else if (isWord(token)) {
- numTokens++;
- numWords++;
- numWordsCurrentLine++;
- if (inAnchorText) {
- numLinkedWords++;
- }
- final int tokenLength = token.length();
- currentLineLength += tokenLength + 1;
- if (currentLineLength > maxLineLength) {
- numWrappedLines++;
- currentLineLength = tokenLength;
- numWordsCurrentLine = 1;
- }
- } else {
- numTokens++;
- }
- }
- if (numTokens == 0) {
- return;
- }
- int numWordsInWrappedLines;
- if (numWrappedLines == 0) {
- numWordsInWrappedLines = numWords;
- numWrappedLines = 1;
- } else {
- numWordsInWrappedLines = numWords - numWordsCurrentLine;
- }
-
- TextBlock tb =
- new TextBlock(textBuffer.toString().trim(), currentContainedTextElements, numWords,
- numLinkedWords, numWordsInWrappedLines, numWrappedLines, offsetBlocks);
- currentContainedTextElements = new BitSet();
-
- offsetBlocks++;
-
- textBuffer.setLength(0);
- tokenBuffer.setLength(0);
-
- tb.setTagLevel(blockTagLevel);
- addTextBlock(tb);
- blockTagLevel = -1;
- }
-
- protected void addTextBlock(final TextBlock tb) {
-
- for (Integer l : fontSizeStack) {
- if (l != null) {
- tb.addLabel("font-" + l);
- break;
- }
- }
- for (LinkedList<LabelAction> labelStack : labelStacks) {
- if (labelStack != null) {
- for (LabelAction labels : labelStack) {
- if (labels != null) {
- labels.addTo(tb);
- }
- }
- }
- }
-
- textBlocks.add(tb);
- }
-
- private static final Pattern PAT_VALID_WORD_CHARACTER = Pattern
- .compile("[\\p{L}\\p{Nd}\\p{Nl}\\p{No}]");
-
- private static boolean isWord(final String token) {
- return PAT_VALID_WORD_CHARACTER.matcher(token).find();
- }
-
- static private enum Event {
- START_TAG, END_TAG, CHARACTERS, WHITESPACE
- }
-
- public String getTitle() {
- return title;
- }
-
- public void setTitle(String s) {
- if (s == null || s.length() == 0) {
- return;
- }
- title = s;
- }
-
- /**
- * Returns a {@link TextDocument} containing the extracted {@link TextBlock} s. NOTE: Only call
- * this after parsing.
- *
- * @return The {@link TextDocument}
- */
- public TextDocument toTextDocument() {
- // just to be sure
- flushBlock();
-
- return new TextDocument(getTitle(), getTextBlocks());
- }
-
- public void addWhitespaceIfNecessary() {
- if (!sbLastWasWhitespace) {
- tokenBuffer.append(' ');
- textBuffer.append(' ');
- sbLastWasWhitespace = true;
- }
- }
-
- public void addLabelAction(final LabelAction la) throws IllegalStateException {
- LinkedList<LabelAction> labelStack = labelStacks.getLast();
- if (labelStack == null) {
- labelStack = new LinkedList<LabelAction>();
- labelStacks.removeLast();
- labelStacks.add(labelStack);
- }
- labelStack.add(la);
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/BoilerpipeHTMLParser.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/BoilerpipeHTMLParser.java
deleted file mode 100644
index f9f07099df..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/BoilerpipeHTMLParser.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import com.kohlschutter.boilerpipe.BoilerpipeDocumentSource;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import org.apache.xerces.parsers.AbstractSAXParser;
-import org.cyberneko.html.HTMLConfiguration;
-
-/**
- * A simple SAX Parser, used by {@link BoilerpipeSAXInput}. The parser uses <a
- * href="http://nekohtml.sourceforge.net/">CyberNeko</a> to parse HTML content.
- */
-public class BoilerpipeHTMLParser extends AbstractSAXParser implements BoilerpipeDocumentSource {
-
- private BoilerpipeHTMLContentHandler contentHandler;
-
- /**
- * Constructs a {@link BoilerpipeHTMLParser} using a default HTML content handler.
- */
- public BoilerpipeHTMLParser() {
- this(new BoilerpipeHTMLContentHandler());
- }
-
- /**
- * Constructs a {@link BoilerpipeHTMLParser} using the given {@link BoilerpipeHTMLContentHandler}.
- *
- * @param contentHandler
- */
- public BoilerpipeHTMLParser(BoilerpipeHTMLContentHandler contentHandler) {
- super(new HTMLConfiguration());
- setContentHandler(contentHandler);
- }
-
- protected BoilerpipeHTMLParser(boolean ignore) {
- super(new HTMLConfiguration());
- }
-
- public void setContentHandler(final BoilerpipeHTMLContentHandler contentHandler) {
- this.contentHandler = contentHandler;
- super.setContentHandler(contentHandler);
- }
-
- public void setContentHandler(final org.xml.sax.ContentHandler contentHandler) {
- this.contentHandler = null;
- super.setContentHandler(contentHandler);
- }
-
- /**
- * Returns a {@link TextDocument} containing the extracted {@link TextBlock} s. NOTE: Only call
- * this after {@link #parse(org.xml.sax.InputSource)}.
- *
- * @return The {@link TextDocument}
- */
- public TextDocument toTextDocument() {
- return contentHandler.toTextDocument();
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/BoilerpipeSAXInput.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/BoilerpipeSAXInput.java
deleted file mode 100644
index 21c7cf63a5..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/BoilerpipeSAXInput.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import com.kohlschutter.boilerpipe.BoilerpipeInput;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-
-/**
- * Parses an {@link InputSource} using SAX and returns a {@link TextDocument}.
- */
-public final class BoilerpipeSAXInput implements BoilerpipeInput {
- private final InputSource is;
-
- /**
- * Creates a new instance of {@link BoilerpipeSAXInput} for the given {@link InputSource}.
- *
- * @param is
- * @throws SAXException
- */
- public BoilerpipeSAXInput(final InputSource is) throws SAXException {
- this.is = is;
- }
-
- /**
- * Retrieves the {@link TextDocument} using a default HTML parser.
- */
- public TextDocument getTextDocument() throws BoilerpipeProcessingException {
- return getTextDocument(new BoilerpipeHTMLParser());
- }
-
- /**
- * Retrieves the {@link TextDocument} using the given HTML parser.
- *
- * @param parser The parser used to transform the input into boilerpipe's internal representation.
- * @return The retrieved {@link TextDocument}
- * @throws BoilerpipeProcessingException
- */
- public TextDocument getTextDocument(final BoilerpipeHTMLParser parser)
- throws BoilerpipeProcessingException {
- try {
- parser.parse(is);
- } catch (IOException e) {
- throw new BoilerpipeProcessingException(e);
- } catch (SAXException e) {
- throw new BoilerpipeProcessingException(e);
- }
-
- return parser.toTextDocument();
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/CommonTagActions.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/CommonTagActions.java
deleted file mode 100644
index 86758bea88..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/CommonTagActions.java
+++ /dev/null
@@ -1,343 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import com.kohlschutter.boilerpipe.labels.LabelAction;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import org.xml.sax.Attributes;
-import org.xml.sax.SAXException;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Defines an action that is to be performed whenever a particular tag occurs during HTML parsing.
- */
-public abstract class CommonTagActions {
-
- private CommonTagActions() {
- }
-
- public static final class Chained implements TagAction {
-
- private final TagAction t1;
- private final TagAction t2;
-
- public Chained(final TagAction t1, final TagAction t2) {
- this.t1 = t1;
- this.t2 = t2;
- }
-
- public boolean start(BoilerpipeHTMLContentHandler instance, String localName, String qName,
- Attributes atts) throws SAXException {
- return t1.start(instance, localName, qName, atts)
- | t2.start(instance, localName, qName, atts);
- }
-
- public boolean end(BoilerpipeHTMLContentHandler instance, String localName, String qName)
- throws SAXException {
- return t1.end(instance, localName, qName) | t2.end(instance, localName, qName);
- }
-
- public boolean changesTagLevel() {
- return t1.changesTagLevel() || t2.changesTagLevel();
- }
- }
-
- /**
- * Marks this tag as "ignorable", i.e. all its inner content is silently skipped.
- */
- public static final TagAction TA_IGNORABLE_ELEMENT = new TagAction() {
-
- public boolean start(final BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName, final Attributes atts) {
- instance.inIgnorableElement++;
- return true;
- }
-
- public boolean end(final BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName) {
- instance.inIgnorableElement--;
- return true;
- }
-
- public boolean changesTagLevel() {
- return true;
- }
- };
-
- /**
- * Marks this tag as "anchor" (this should usually only be set for the <code><A></code>
- * tag). Anchor tags may not be nested.
- *
- * There is a bug in certain versions of NekoHTML which still allows nested tags. If boilerpipe
- * encounters such nestings, a SAXException is thrown.
- */
- public static final TagAction TA_ANCHOR_TEXT = new TagAction() {
-
- public boolean start(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName, final Attributes atts) throws SAXException {
- if (instance.inAnchor++ > 0) {
- // as nested A elements are not allowed per specification, we
- // are probably reaching this branch due to a bug in the XML
- // parser
- System.err
- .println(
- "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to boilerpipe again. Trying to recover somehow...");
-
- end(instance, localName, qName);
- }
- if (instance.inIgnorableElement == 0) {
- instance.addWhitespaceIfNecessary();
- instance.tokenBuffer.append(BoilerpipeHTMLContentHandler.ANCHOR_TEXT_START);
- instance.tokenBuffer.append(' ');
- instance.sbLastWasWhitespace = true;
- }
- return false;
- }
-
- public boolean end(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName) {
- if (--instance.inAnchor == 0) {
- if (instance.inIgnorableElement == 0) {
- instance.addWhitespaceIfNecessary();
- instance.tokenBuffer.append(BoilerpipeHTMLContentHandler.ANCHOR_TEXT_END);
- instance.tokenBuffer.append(' ');
- instance.sbLastWasWhitespace = true;
- }
- }
- return false;
- }
-
- public boolean changesTagLevel() {
- return true;
- }
- };
-
- /**
- * Marks this tag the body element (this should usually only be set for the
- * <code><BODY></code> tag).
- */
- public static final TagAction TA_BODY = new TagAction() {
- public boolean start(final BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName, final Attributes atts) {
- instance.flushBlock();
- instance.inBody++;
- return false;
- }
-
- public boolean end(final BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName) {
- instance.flushBlock();
- instance.inBody--;
- return false;
- }
-
- public boolean changesTagLevel() {
- return true;
- }
- };
-
- /**
- * Marks this tag a simple "inline" element, which generates whitespace, but no new block.
- */
- public static final TagAction TA_INLINE_WHITESPACE = new TagAction() {
-
- public boolean start(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName, final Attributes atts) {
- instance.addWhitespaceIfNecessary();
- return false;
- }
-
- public boolean end(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName) {
- instance.addWhitespaceIfNecessary();
- return false;
- }
-
- public boolean changesTagLevel() {
- return false;
- }
- };
-
- /**
- * @deprecated Use {@link #TA_INLINE_WHITESPACE} instead
- */
- @Deprecated
- public static final TagAction TA_INLINE = TA_INLINE_WHITESPACE;
-
- /**
- * Marks this tag a simple "inline" element, which neither generates whitespace, nor a new block.
- */
- public static final TagAction TA_INLINE_NO_WHITESPACE = new TagAction() {
-
- public boolean start(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName, final Attributes atts) {
- return false;
- }
-
- public boolean end(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName) {
- return false;
- }
-
- public boolean changesTagLevel() {
- return false;
- }
- };
- private static final Pattern PAT_FONT_SIZE = Pattern.compile("([\\+\\-]?)([0-9])");
-
- /**
- * Explicitly marks this tag a simple "block-level" element, which always generates whitespace
- */
- public static final TagAction TA_BLOCK_LEVEL = new TagAction() {
-
- public boolean start(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName, final Attributes atts) {
- return true;
- }
-
- public boolean end(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName) {
- return true;
- }
-
- public boolean changesTagLevel() {
- return true;
- }
- };
-
- /**
- * Special TagAction for the <code><FONT></code> tag, which keeps track of the absolute and
- * relative font size.
- */
- public static final TagAction TA_FONT = new TagAction() {
-
- public boolean start(final BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName, final Attributes atts) {
-
- String sizeAttr = atts.getValue("size");
- if (sizeAttr != null) {
- Matcher m = PAT_FONT_SIZE.matcher(sizeAttr);
- if (m.matches()) {
- String rel = m.group(1);
- final int val = Integer.parseInt(m.group(2));
- final int size;
- if (rel.length() == 0) {
- // absolute
- size = val;
- } else {
- // relative
- int prevSize;
- if (instance.fontSizeStack.isEmpty()) {
- prevSize = 3;
- } else {
- prevSize = 3;
- for (Integer s : instance.fontSizeStack) {
- if (s != null) {
- prevSize = s;
- break;
- }
- }
- }
- if (rel.charAt(0) == '+') {
- size = prevSize + val;
- } else {
- size = prevSize - val;
- }
-
- }
- instance.fontSizeStack.add(0, size);
- } else {
- instance.fontSizeStack.add(0, null);
- }
- } else {
- instance.fontSizeStack.add(0, null);
- }
- return false;
- }
-
- public boolean end(final BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName) {
- instance.fontSizeStack.removeFirst();
- return false;
- }
-
- public boolean changesTagLevel() {
- return false;
- }
- };
-
- /**
- * {@link CommonTagActions} for inline elements, which triggers some {@link LabelAction} on the
- * generated {@link TextBlock}.
- */
- public static final class InlineTagLabelAction implements TagAction {
-
- private final LabelAction action;
-
- public InlineTagLabelAction(final LabelAction action) {
- this.action = action;
- }
-
- public boolean start(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName, final Attributes atts) {
- instance.addWhitespaceIfNecessary();
- instance.addLabelAction(action);
- return false;
- }
-
- public boolean end(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName) {
- instance.addWhitespaceIfNecessary();
- return false;
- }
-
- public boolean changesTagLevel() {
- return false;
- }
- }
-
- /**
- * {@link CommonTagActions} for block-level elements, which triggers some {@link LabelAction} on
- * the generated {@link TextBlock}.
- */
- public static final class BlockTagLabelAction implements TagAction {
-
- private final LabelAction action;
-
- public BlockTagLabelAction(final LabelAction action) {
- this.action = action;
- }
-
- public boolean start(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName, final Attributes atts) {
- instance.addLabelAction(action);
- return true;
- }
-
- public boolean end(BoilerpipeHTMLContentHandler instance, final String localName,
- final String qName) {
- return true;
- }
-
- public boolean changesTagLevel() {
- return true;
- }
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/DefaultTagActionMap.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/DefaultTagActionMap.java
deleted file mode 100644
index 41d3c48305..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/DefaultTagActionMap.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-import com.kohlschutter.boilerpipe.labels.LabelAction;
-
-/**
- * Default {@link TagAction}s. Seem to work well.
- */
-public class DefaultTagActionMap extends TagActionMap {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public static final TagActionMap INSTANCE = new DefaultTagActionMap();
-
- protected DefaultTagActionMap() {
- setTagAction("STYLE", CommonTagActions.TA_IGNORABLE_ELEMENT);
- setTagAction("SCRIPT", CommonTagActions.TA_IGNORABLE_ELEMENT);
- setTagAction("OPTION", CommonTagActions.TA_IGNORABLE_ELEMENT);
- setTagAction("OBJECT", CommonTagActions.TA_IGNORABLE_ELEMENT);
- setTagAction("EMBED", CommonTagActions.TA_IGNORABLE_ELEMENT);
- setTagAction("APPLET", CommonTagActions.TA_IGNORABLE_ELEMENT);
- setTagAction("LINK", CommonTagActions.TA_IGNORABLE_ELEMENT);
-
- setTagAction("A", CommonTagActions.TA_ANCHOR_TEXT);
- setTagAction("BODY", CommonTagActions.TA_BODY);
-
- setTagAction("STRIKE", CommonTagActions.TA_INLINE_NO_WHITESPACE);
- setTagAction("U", CommonTagActions.TA_INLINE_NO_WHITESPACE);
- setTagAction("B", CommonTagActions.TA_INLINE_NO_WHITESPACE);
- setTagAction("I", CommonTagActions.TA_INLINE_NO_WHITESPACE);
- setTagAction("EM", CommonTagActions.TA_INLINE_NO_WHITESPACE);
- setTagAction("STRONG", CommonTagActions.TA_INLINE_NO_WHITESPACE);
- setTagAction("SPAN", CommonTagActions.TA_INLINE_NO_WHITESPACE);
-
- // New in 1.1 (especially to improve extraction quality from Wikipedia etc.)
- setTagAction("SUP", CommonTagActions.TA_INLINE_NO_WHITESPACE);
-
- // New in 1.2
- setTagAction("CODE", CommonTagActions.TA_INLINE_NO_WHITESPACE);
- setTagAction("TT", CommonTagActions.TA_INLINE_NO_WHITESPACE);
- setTagAction("SUB", CommonTagActions.TA_INLINE_NO_WHITESPACE);
- setTagAction("VAR", CommonTagActions.TA_INLINE_NO_WHITESPACE);
-
- setTagAction("ABBR", CommonTagActions.TA_INLINE_WHITESPACE);
- setTagAction("ACRONYM", CommonTagActions.TA_INLINE_WHITESPACE);
-
- setTagAction("FONT", CommonTagActions.TA_INLINE_NO_WHITESPACE); // could also use TA_FONT
-
- // added in 1.1.1
- setTagAction("NOSCRIPT", CommonTagActions.TA_IGNORABLE_ELEMENT);
-
- // New in 1.3
- setTagAction("LI", new CommonTagActions.BlockTagLabelAction(new LabelAction(DefaultLabels.LI)));
- setTagAction("H1", new CommonTagActions.BlockTagLabelAction(new LabelAction(DefaultLabels.H1,
- DefaultLabels.HEADING)));
- setTagAction("H2", new CommonTagActions.BlockTagLabelAction(new LabelAction(DefaultLabels.H2,
- DefaultLabels.HEADING)));
- setTagAction("H3", new CommonTagActions.BlockTagLabelAction(new LabelAction(DefaultLabels.H3,
- DefaultLabels.HEADING)));
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/HTMLDocument.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/HTMLDocument.java
deleted file mode 100644
index 8ec1f61178..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/HTMLDocument.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import org.xml.sax.InputSource;
-
-import java.io.ByteArrayInputStream;
-import java.nio.charset.Charset;
-
-/**
- * An {@link InputSourceable} for {@link HTMLFetcher}.
- */
-public class HTMLDocument implements InputSourceable {
- private final Charset charset;
- private final byte[] data;
-
- public HTMLDocument(final byte[] data, final Charset charset) {
- this.data = data;
- this.charset = charset;
- }
-
- public HTMLDocument(final String data) {
- Charset cs = Charset.forName("utf-8");
- this.data = data.getBytes(cs);
- this.charset = cs;
- }
-
- public Charset getCharset() {
- return charset;
- }
-
- public byte[] getData() {
- return data;
- }
-
- public InputSource toInputSource() {
- final InputSource is = new InputSource(new ByteArrayInputStream(data));
- is.setEncoding(charset.name());
- return is;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/HTMLFetcher.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/HTMLFetcher.java
deleted file mode 100644
index 55c31104dd..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/HTMLFetcher.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.net.URLConnection;
-import java.nio.charset.Charset;
-import java.nio.charset.UnsupportedCharsetException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPInputStream;
-
-/**
- * A very simple HTTP/HTML fetcher, really just for demo purposes.
- */
-public class HTMLFetcher {
- private HTMLFetcher() {
- }
-
- private static final Pattern PAT_CHARSET = Pattern.compile("charset=([^; ]+)$");
-
- /**
- * Fetches the document at the given URL, using {@link URLConnection}.
- *
- * @param url
- * @return
- * @throws IOException
- */
- public static HTMLDocument fetch(final URL url) throws IOException {
- final URLConnection conn = url.openConnection();
- final String ct = conn.getContentType();
-
- if (ct == null || !(ct.equals("text/html") || ct.startsWith("text/html;"))) {
- throw new IOException("Unsupported content type: " + ct);
- }
-
- Charset cs = Charset.forName("Cp1252");
- if (ct != null) {
- Matcher m = PAT_CHARSET.matcher(ct);
- if (m.find()) {
- final String charset = m.group(1);
- try {
- cs = Charset.forName(charset);
- } catch (UnsupportedCharsetException e) {
- // keep default
- }
- }
- }
-
- InputStream in = conn.getInputStream();
-
- final String encoding = conn.getContentEncoding();
- if (encoding != null) {
- if ("gzip".equalsIgnoreCase(encoding)) {
- in = new GZIPInputStream(in);
- } else {
- System.err.println("WARN: unsupported Content-Encoding: " + encoding);
- }
- }
-
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- byte[] buf = new byte[4096];
- int r;
- while ((r = in.read(buf)) != -1) {
- bos.write(buf, 0, r);
- }
- in.close();
-
- final byte[] data = bos.toByteArray();
-
- return new HTMLDocument(data, cs);
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/HTMLHighlighter.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/HTMLHighlighter.java
deleted file mode 100644
index 27263047f3..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/HTMLHighlighter.java
+++ /dev/null
@@ -1,504 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import com.kohlschutter.boilerpipe.BoilerpipeExtractor;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import org.apache.xerces.parsers.AbstractSAXParser;
-import org.cyberneko.html.HTMLConfiguration;
-import org.xml.sax.Attributes;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.InputSource;
-import org.xml.sax.Locator;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.io.StringReader;
-import java.net.URL;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Highlights text blocks in an HTML document that have been marked as "content" in the
- * corresponding {@link TextDocument}.
- */
-public final class HTMLHighlighter implements Serializable {
-
- private Map<String, Set<String>> tagWhitelist = null;
-
- /**
- * Creates a new {@link HTMLHighlighter}, which is set-up to return the full HTML text, with the
- * extracted text portion <b>highlighted</b>.
- */
- public static HTMLHighlighter newHighlightingInstance() {
- return new HTMLHighlighter(false);
- }
-
- /**
- * Creates a new {@link HTMLHighlighter}, which is set-up to return only the extracted HTML text,
- * including enclosed markup.
- */
- public static HTMLHighlighter newExtractingInstance() {
- return new HTMLHighlighter(true);
- }
-
- private HTMLHighlighter(final boolean extractHTML) {
- if (extractHTML) {
- setOutputHighlightOnly(true);
- setExtraStyleSheet("\n<style type=\"text/css\">\n" + "A:before { content:' '; } \n" //
- + "A:after { content:' '; } \n" //
- + "SPAN:before { content:' '; } \n" //
- + "SPAN:after { content:' '; } \n" //
- + "</style>\n");
- setPreHighlight("");
- setPostHighlight("");
- }
- }
-
- /**
- * Processes the given {@link TextDocument} and the original HTML text (as a String).
- *
- * @param doc The processed {@link TextDocument}.
- * @param origHTML The original HTML document.
- * @return The highlighted HTML.
- * @throws BoilerpipeProcessingException
- */
- public String process(final TextDocument doc, final String origHTML)
- throws BoilerpipeProcessingException {
- return process(doc, new InputSource(new StringReader(origHTML)));
- }
-
- /**
- * Processes the given {@link TextDocument} and the original HTML text (as an {@link InputSource}
- * ).
- *
- * @param doc The processed {@link TextDocument}.
- * @param is The original HTML document.
- * @return The highlighted HTML.
- * @throws BoilerpipeProcessingException
- */
- public String process(final TextDocument doc, final InputSource is)
- throws BoilerpipeProcessingException {
- final Implementation implementation = new Implementation();
- implementation.process(doc, is);
-
- String html = implementation.html.toString();
- if (outputHighlightOnly) {
- Matcher m;
-
- boolean repeat = true;
- while (repeat) {
- repeat = false;
- m = PAT_TAG_NO_TEXT.matcher(html);
- if (m.find()) {
- repeat = true;
- html = m.replaceAll("");
- }
-
- m = PAT_SUPER_TAG.matcher(html);
- if (m.find()) {
- repeat = true;
- html = m.replaceAll(m.group(1));
- }
- }
- }
-
- return html;
- }
-
- private static final Pattern PAT_TAG_NO_TEXT = Pattern.compile("<[^/][^>]*></[^>]*>");
- private static final Pattern PAT_SUPER_TAG = Pattern.compile("^<[^>]*>(<.*?>)</[^>]*>$");
-
- /**
- * Fetches the given {@link URL} using {@link HTMLFetcher} and processes the retrieved HTML using
- * the specified {@link BoilerpipeExtractor}.
- *
- * @param doc The processed {@link TextDocument}.
- * @param is The original HTML document.
- * @return The highlighted HTML.
- * @throws BoilerpipeProcessingException
- */
- public String process(final URL url, final BoilerpipeExtractor extractor) throws IOException,
- BoilerpipeProcessingException, SAXException {
- final HTMLDocument htmlDoc = HTMLFetcher.fetch(url);
-
- final TextDocument doc = new BoilerpipeSAXInput(htmlDoc.toInputSource()).getTextDocument();
- extractor.process(doc);
-
- final InputSource is = htmlDoc.toInputSource();
-
- return process(doc, is);
- }
-
- private boolean outputHighlightOnly = false;
- private String extraStyleSheet = "\n<style type=\"text/css\">\n" + ".x-boilerpipe-mark1 {"
- + " text-decoration:none; " + "background-color: #ffff42 !important; "
- + "color: black !important; " + "display:inline !important; "
- + "visibility:visible !important; }\n" + //
- "</style>\n";
- private String preHighlight = "<span class=\"x-boilerpipe-mark1\">";
- private String postHighlight = "</span>";
-
- /**
- * If true, only HTML enclosed within highlighted content will be returned
- */
- public boolean isOutputHighlightOnly() {
- return outputHighlightOnly;
- }
-
- /**
- * Sets whether only HTML enclosed within highlighted content will be returned, or the whole HTML
- * document.
- */
- public void setOutputHighlightOnly(boolean outputHighlightOnly) {
- this.outputHighlightOnly = outputHighlightOnly;
- }
-
- /**
- * Returns the extra stylesheet definition that will be inserted in the HEAD element.
- *
- * By default, this corresponds to a simple definition that marks text in class
- * "x-boilerpipe-mark1" as inline text with yellow background.
- */
- public String getExtraStyleSheet() {
- return extraStyleSheet;
- }
-
- /**
- * Sets the extra stylesheet definition that will be inserted in the HEAD element.
- *
- * To disable, set it to the empty string: ""
- *
- * @param extraStyleSheet Plain HTML
- */
- public void setExtraStyleSheet(String extraStyleSheet) {
- this.extraStyleSheet = extraStyleSheet;
- }
-
- /**
- * Returns the string that will be inserted before any highlighted HTML block.
- *
- * By default, this corresponds to <code><span class=&qupt;x-boilerpipe-mark1"></code>
- */
- public String getPreHighlight() {
- return preHighlight;
- }
-
- /**
- * Sets the string that will be inserted prior to any highlighted HTML block.
- *
- * To disable, set it to the empty string: ""
- */
- public void setPreHighlight(String preHighlight) {
- this.preHighlight = preHighlight;
- }
-
- /**
- * Returns the string that will be inserted after any highlighted HTML block.
- *
- * By default, this corresponds to <code></span></code>
- */
- public String getPostHighlight() {
- return postHighlight;
- }
-
- /**
- * Sets the string that will be inserted after any highlighted HTML block.
- *
- * To disable, set it to the empty string: ""
- */
- public void setPostHighlight(String postHighlight) {
- this.postHighlight = postHighlight;
- }
-
- private abstract static class TagAction {
- void beforeStart(final Implementation instance, final String localName) {
- }
-
- void afterStart(final Implementation instance, final String localName) {
- }
-
- void beforeEnd(final Implementation instance, final String localName) {
- }
-
- void afterEnd(final Implementation instance, final String localName) {
- }
- }
-
- private static final TagAction TA_IGNORABLE_ELEMENT = new TagAction() {
- void beforeStart(final Implementation instance, final String localName) {
- instance.inIgnorableElement++;
- }
-
- void afterEnd(final Implementation instance, final String localName) {
- instance.inIgnorableElement--;
- }
- };
-
- private static final TagAction TA_HEAD = new TagAction() {
- void beforeStart(final Implementation instance, final String localName) {
- instance.inIgnorableElement++;
- }
-
- void beforeEnd(final Implementation instance, String localName) {
- instance.html.append(instance.hl.extraStyleSheet);
- }
-
- void afterEnd(final Implementation instance, final String localName) {
- instance.inIgnorableElement--;
- }
- };
- private static Map<String, TagAction> TAG_ACTIONS = new HashMap<String, TagAction>();
-
- static {
- TAG_ACTIONS.put("STYLE", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("SCRIPT", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("OPTION", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("NOSCRIPT", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("OBJECT", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("EMBED", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("APPLET", TA_IGNORABLE_ELEMENT);
- // NOTE: you might want to comment this out:
- TAG_ACTIONS.put("LINK", TA_IGNORABLE_ELEMENT);
-
- TAG_ACTIONS.put("HEAD", TA_HEAD);
- }
-
- private final class Implementation extends AbstractSAXParser implements ContentHandler {
- StringBuilder html = new StringBuilder();
-
- private int inIgnorableElement = 0;
- private int characterElementIdx = 0;
- private final BitSet contentBitSet = new BitSet();
- private final HTMLHighlighter hl = HTMLHighlighter.this;
-
- Implementation() {
- super(new HTMLConfiguration());
- setContentHandler(this);
- }
-
- void process(final TextDocument doc, final InputSource is) throws BoilerpipeProcessingException {
- for (TextBlock block : doc.getTextBlocks()) {
- if (block.isContent()) {
- final BitSet bs = block.getContainedTextElements();
- if (bs != null) {
- contentBitSet.or(bs);
- }
- }
- }
-
- try {
- parse(is);
- } catch (SAXException e) {
- throw new BoilerpipeProcessingException(e);
- } catch (IOException e) {
- throw new BoilerpipeProcessingException(e);
- }
- }
-
- public void endDocument() throws SAXException {
- }
-
- public void endPrefixMapping(String prefix) throws SAXException {
- }
-
- public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException {
- }
-
- public void processingInstruction(String target, String data) throws SAXException {
- }
-
- public void setDocumentLocator(Locator locator) {
- }
-
- public void skippedEntity(String name) throws SAXException {
- }
-
- public void startDocument() throws SAXException {
- }
-
- public void startElement(String uri, String localName, String qName, Attributes atts)
- throws SAXException {
- TagAction ta = TAG_ACTIONS.get(localName);
- if (ta != null) {
- ta.beforeStart(this, localName);
- }
-
- // HACK: remove existing highlight
- boolean ignoreAttrs = false;
- if ("SPAN".equalsIgnoreCase(localName)) {
- String classVal = atts.getValue("class");
- if ("x-boilerpipe-mark1".equals(classVal)) {
- ignoreAttrs = true;
- }
- }
-
- try {
- if (inIgnorableElement == 0) {
- if (outputHighlightOnly) {
- // boolean highlight = contentBitSet
- // .get(characterElementIdx);
-
- // if (!highlight) {
- // return;
- // }
- }
-
- final Set<String> whitelistAttributes;
- if (tagWhitelist == null) {
- whitelistAttributes = null;
- } else {
- whitelistAttributes = tagWhitelist.get(qName);
- if (whitelistAttributes == null) {
- // skip
- return;
- }
- }
-
- html.append('<');
- html.append(qName);
- if (!ignoreAttrs) {
- final int numAtts = atts.getLength();
- for (int i = 0; i < numAtts; i++) {
- final String attr = atts.getQName(i);
-
- if (whitelistAttributes != null && !whitelistAttributes.contains(attr)) {
- // skip
- continue;
- }
-
- final String value = atts.getValue(i);
- html.append(' ');
- html.append(attr);
- html.append("=\"");
- html.append(xmlEncode(value));
- html.append("\"");
- }
- }
- html.append('>');
- }
- } finally {
- if (ta != null) {
- ta.afterStart(this, localName);
- }
- }
- }
-
- public void endElement(String uri, String localName, String qName) throws SAXException {
- TagAction ta = TAG_ACTIONS.get(localName);
- if (ta != null) {
- ta.beforeEnd(this, localName);
- }
-
- try {
- if (inIgnorableElement == 0) {
- if (outputHighlightOnly) {
- // boolean highlight = contentBitSet
- // .get(characterElementIdx);
-
- // if (!highlight) {
- // return;
- // }
- }
-
- if (tagWhitelist != null && !tagWhitelist.containsKey(qName)) {
- // skip
- return;
- }
-
- html.append("</");
- html.append(qName);
- html.append('>');
- }
- } finally {
- if (ta != null) {
- ta.afterEnd(this, localName);
- }
- }
- }
-
- public void characters(char[] ch, int start, int length) throws SAXException {
- characterElementIdx++;
- if (inIgnorableElement == 0) {
-
- boolean highlight = contentBitSet.get(characterElementIdx);
-
- if (!highlight && outputHighlightOnly) {
- return;
- }
-
- if (highlight) {
- html.append(preHighlight);
- }
- html.append(xmlEncode(String.valueOf(ch, start, length)));
- if (highlight) {
- html.append(postHighlight);
- }
- }
- }
-
- public void startPrefixMapping(String prefix, String uri) throws SAXException {
- }
-
- }
-
- private static String xmlEncode(final String in) {
- if (in == null) {
- return "";
- }
- char c;
- StringBuilder out = new StringBuilder(in.length());
-
- for (int i = 0; i < in.length(); i++) {
- c = in.charAt(i);
- switch (c) {
- case '<':
- out.append("<");
- break;
- case '>':
- out.append(">");
- break;
- case '&':
- out.append("&");
- break;
- case '"':
- out.append(""");
- break;
- default:
- out.append(c);
- }
- }
-
- return out.toString();
- }
-
- public Map<String, Set<String>> getTagWhitelist() {
- return tagWhitelist;
- }
-
- public void setTagWhitelist(Map<String, Set<String>> tagWhitelist) {
- this.tagWhitelist = tagWhitelist;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/ImageExtractor.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/ImageExtractor.java
deleted file mode 100644
index 6466d04de4..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/ImageExtractor.java
+++ /dev/null
@@ -1,278 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import com.kohlschutter.boilerpipe.BoilerpipeExtractor;
-import com.kohlschutter.boilerpipe.BoilerpipeProcessingException;
-import com.kohlschutter.boilerpipe.document.Image;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import com.kohlschutter.boilerpipe.document.TextDocument;
-import org.apache.xerces.parsers.AbstractSAXParser;
-import org.cyberneko.html.HTMLConfiguration;
-import org.xml.sax.Attributes;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.InputSource;
-import org.xml.sax.Locator;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.io.StringReader;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Extracts the images that are enclosed by extracted content.
- */
-public final class ImageExtractor {
- public static final ImageExtractor INSTANCE = new ImageExtractor();
-
- /**
- * Returns the singleton instance of {@link ImageExtractor}.
- *
- * @return
- */
- public static ImageExtractor getInstance() {
- return INSTANCE;
- }
-
- private ImageExtractor() {
- }
-
- /**
- * Processes the given {@link TextDocument} and the original HTML text (as a String).
- *
- * @param doc The processed {@link TextDocument}.
- * @param origHTML The original HTML document.
- * @return A List of enclosed {@link Image}s
- * @throws BoilerpipeProcessingException
- */
- public List<Image> process(final TextDocument doc, final String origHTML)
- throws BoilerpipeProcessingException {
- return process(doc, new InputSource(new StringReader(origHTML)));
- }
-
- /**
- * Processes the given {@link TextDocument} and the original HTML text (as an {@link InputSource}
- * ).
- *
- * @param doc The processed {@link TextDocument}.
- * @param origHTML The original HTML document.
- * @return A List of enclosed {@link Image}s
- * @throws BoilerpipeProcessingException
- */
- public List<Image> process(final TextDocument doc, final InputSource is)
- throws BoilerpipeProcessingException {
- final Implementation implementation = new Implementation();
- implementation.process(doc, is);
-
- return implementation.linksHighlight;
- }
-
- /**
- * Fetches the given {@link URL} using {@link HTMLFetcher} and processes the retrieved HTML using
- * the specified {@link BoilerpipeExtractor}.
- *
- * @param doc The processed {@link TextDocument}.
- * @param is The original HTML document.
- * @return A List of enclosed {@link Image}s
- * @throws BoilerpipeProcessingException
- */
- public List<Image> process(final URL url, final BoilerpipeExtractor extractor)
- throws IOException, BoilerpipeProcessingException, SAXException {
- final HTMLDocument htmlDoc = HTMLFetcher.fetch(url);
-
- final TextDocument doc = new BoilerpipeSAXInput(htmlDoc.toInputSource()).getTextDocument();
- extractor.process(doc);
-
- final InputSource is = htmlDoc.toInputSource();
-
- return process(doc, is);
- }
-
- private final class Implementation extends AbstractSAXParser implements ContentHandler {
- List<Image> linksHighlight = new ArrayList<Image>();
- private List<Image> linksBuffer = new ArrayList<Image>();
-
- private int inIgnorableElement = 0;
- private int characterElementIdx = 0;
- private final BitSet contentBitSet = new BitSet();
-
- private boolean inHighlight = false;
-
- Implementation() {
- super(new HTMLConfiguration());
- setContentHandler(this);
- }
-
- void process(final TextDocument doc, final InputSource is) throws BoilerpipeProcessingException {
- for (TextBlock block : doc.getTextBlocks()) {
- if (block.isContent()) {
- final BitSet bs = block.getContainedTextElements();
- if (bs != null) {
- contentBitSet.or(bs);
- }
- }
- }
-
- try {
- parse(is);
- } catch (SAXException e) {
- throw new BoilerpipeProcessingException(e);
- } catch (IOException e) {
- throw new BoilerpipeProcessingException(e);
- }
- }
-
- public void endDocument() throws SAXException {
- }
-
- public void endPrefixMapping(String prefix) throws SAXException {
- }
-
- public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException {
- }
-
- public void processingInstruction(String target, String data) throws SAXException {
- }
-
- public void setDocumentLocator(Locator locator) {
- }
-
- public void skippedEntity(String name) throws SAXException {
- }
-
- public void startDocument() throws SAXException {
- }
-
- public void startElement(String uri, String localName, String qName, Attributes atts)
- throws SAXException {
- TagAction ta = TAG_ACTIONS.get(localName);
- if (ta != null) {
- ta.beforeStart(this, localName);
- }
-
- try {
- if (inIgnorableElement == 0) {
- if (inHighlight && "IMG".equalsIgnoreCase(localName)) {
- String src = atts.getValue("src");
- if (src != null && src.length() > 0) {
- linksBuffer.add(new Image(src, atts.getValue("width"), atts.getValue("height"), atts
- .getValue("alt")));
- }
- }
- }
- } finally {
- if (ta != null) {
- ta.afterStart(this, localName);
- }
- }
- }
-
- public void endElement(String uri, String localName, String qName) throws SAXException {
- TagAction ta = TAG_ACTIONS.get(localName);
- if (ta != null) {
- ta.beforeEnd(this, localName);
- }
-
- try {
- if (inIgnorableElement == 0) {
- //
- }
- } finally {
- if (ta != null) {
- ta.afterEnd(this, localName);
- }
- }
- }
-
- public void characters(char[] ch, int start, int length) throws SAXException {
- characterElementIdx++;
- if (inIgnorableElement == 0) {
-
- boolean highlight = contentBitSet.get(characterElementIdx);
- if (!highlight) {
- if (length == 0) {
- return;
- }
- boolean justWhitespace = true;
- for (int i = start; i < start + length; i++) {
- if (!Character.isWhitespace(ch[i])) {
- justWhitespace = false;
- break;
- }
- }
- if (justWhitespace) {
- return;
- }
- }
-
- inHighlight = highlight;
- if (inHighlight) {
- linksHighlight.addAll(linksBuffer);
- linksBuffer.clear();
- }
- }
- }
-
- public void startPrefixMapping(String prefix, String uri) throws SAXException {
- }
-
- }
-
- private static final TagAction TA_IGNORABLE_ELEMENT = new TagAction() {
- void beforeStart(final Implementation instance, final String localName) {
- instance.inIgnorableElement++;
- }
-
- void afterEnd(final Implementation instance, final String localName) {
- instance.inIgnorableElement--;
- }
- };
-
- private static Map<String, TagAction> TAG_ACTIONS = new HashMap<String, TagAction>();
-
- static {
- TAG_ACTIONS.put("STYLE", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("SCRIPT", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("OPTION", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("NOSCRIPT", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("EMBED", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("APPLET", TA_IGNORABLE_ELEMENT);
- TAG_ACTIONS.put("LINK", TA_IGNORABLE_ELEMENT);
-
- TAG_ACTIONS.put("HEAD", TA_IGNORABLE_ELEMENT);
- }
-
- private abstract static class TagAction {
- void beforeStart(final Implementation instance, final String localName) {
- }
-
- void afterStart(final Implementation instance, final String localName) {
- }
-
- void beforeEnd(final Implementation instance, final String localName) {
- }
-
- void afterEnd(final Implementation instance, final String localName) {
- }
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/InputSourceable.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/InputSourceable.java
deleted file mode 100644
index c4982d90a9..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/InputSourceable.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import org.xml.sax.InputSource;
-
-/**
- * An InputSourceable can return an arbitrary number of new {@link InputSource}s for a given
- * document.
- */
-public interface InputSourceable {
- InputSource toInputSource();
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/MarkupTagAction.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/MarkupTagAction.java
deleted file mode 100644
index 6e8adf07aa..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/MarkupTagAction.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import com.kohlschutter.boilerpipe.labels.DefaultLabels;
-import com.kohlschutter.boilerpipe.labels.LabelAction;
-import com.kohlschutter.boilerpipe.document.TextBlock;
-import org.xml.sax.Attributes;
-import org.xml.sax.SAXException;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-/**
- * Assigns labels for element CSS classes and ids to the corresponding {@link TextBlock}. CSS
- * classes are prefixed by <code>{@link DefaultLabels#MARKUP_PREFIX}.</code>, and IDs are prefixed
- * by <code>{@link DefaultLabels#MARKUP_PREFIX}#</code>
- */
-public final class MarkupTagAction implements TagAction {
-
- private final boolean isBlockLevel;
- private LinkedList<List<String>> labelStack = new LinkedList<List<String>>();
-
- public MarkupTagAction(final boolean isBlockLevel) {
- this.isBlockLevel = isBlockLevel;
- }
-
- private static final Pattern PAT_NUM = Pattern.compile("[0-9]+");
-
- @Override
- public boolean start(BoilerpipeHTMLContentHandler instance, String localName, String qName,
- Attributes atts) throws SAXException {
- List<String> labels = new ArrayList<String>(5);
- labels.add(DefaultLabels.MARKUP_PREFIX + localName);
-
- String classVal = atts.getValue("class");
-
- if (classVal != null && classVal.length() > 0) {
- classVal = PAT_NUM.matcher(classVal).replaceAll("#");
- classVal = classVal.trim();
- String[] vals = classVal.split("[ ]+");
- labels.add(DefaultLabels.MARKUP_PREFIX + "." + classVal.replace(' ', '.'));
- if (vals.length > 1) {
- for (String s : vals) {
- labels.add(DefaultLabels.MARKUP_PREFIX + "." + s);
- }
- }
- }
-
- String id = atts.getValue("id");
- if (id != null && id.length() > 0) {
- id = PAT_NUM.matcher(id).replaceAll("#");
- labels.add(DefaultLabels.MARKUP_PREFIX + "#" + id);
- }
-
- Set<String> ancestors = getAncestorLabels();
- List<String> labelsWithAncestors =
- new ArrayList<String>((ancestors.size() + 1) * labels.size());
-
- for (String l : labels) {
- for (String an : ancestors) {
- labelsWithAncestors.add(an);
- labelsWithAncestors.add(an + " " + l);
- }
- labelsWithAncestors.add(l);
- }
-
- instance.addLabelAction(new LabelAction(labelsWithAncestors
- .toArray(new String[labelsWithAncestors.size()])));
-
- labelStack.add(labels);
-
- return isBlockLevel;
- }
-
- @Override
- public boolean end(BoilerpipeHTMLContentHandler instance, String localName, String qName)
- throws SAXException {
-
- labelStack.removeLast();
- return isBlockLevel;
- }
-
- public boolean changesTagLevel() {
- return isBlockLevel;
- }
-
- private Set<String> getAncestorLabels() {
- Set<String> set = new HashSet<String>();
- for (List<String> labels : labelStack) {
- if (labels == null) {
- continue;
- }
- set.addAll(labels);
- }
- return set;
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/TagAction.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/TagAction.java
deleted file mode 100644
index 880b2170f6..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/TagAction.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import org.xml.sax.Attributes;
-import org.xml.sax.SAXException;
-
-/**
- * Defines an action that is to be performed whenever a particular tag occurs during HTML parsing.
- */
-public interface TagAction {
-
- boolean start(BoilerpipeHTMLContentHandler instance,
- String localName,
- String qName,
- Attributes atts) throws SAXException;
-
- boolean end(BoilerpipeHTMLContentHandler instance,
- String localName,
- String qName) throws SAXException;
-
- boolean changesTagLevel();
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/TagActionMap.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/TagActionMap.java
deleted file mode 100644
index 693719f3be..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/TagActionMap.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.sax;
-
-import java.util.HashMap;
-
-/**
- * Base class for definition a set of {@link TagAction}s that are to be used for the HTML parsing
- * process.
- *
- * @see DefaultTagActionMap
- */
-public abstract class TagActionMap extends HashMap<String, TagAction> {
- private static final long serialVersionUID = 1L;
-
- /**
- * Sets a particular {@link TagAction} for a given tag. Any existing TagAction for that tag will
- * be removed and overwritten.
- *
- * @param tag The tag (will be stored internally 1. as it is, 2. lower-case, 3. upper-case)
- * @param action The {@link TagAction}
- */
- protected void setTagAction(final String tag, final TagAction action) {
- put(tag.toUpperCase(), action);
- put(tag.toLowerCase(), action);
- put(tag, action);
- }
-
- /**
- * Adds a particular {@link TagAction} for a given tag. If a TagAction already exists for that
- * tag, a chained action, consisting of the previous and the new {@link TagAction} is created.
- *
- * @param tag The tag (will be stored internally 1. as it is, 2. lower-case, 3. upper-case)
- * @param action The {@link TagAction}
- */
- protected void addTagAction(final String tag, final TagAction action) {
- TagAction previousAction = get(tag);
- if (previousAction == null) {
- setTagAction(tag, action);
- } else {
- setTagAction(tag, new CommonTagActions.Chained(previousAction, action));
- }
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/package-info.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/package-info.java
deleted file mode 100644
index 4db9c0fea9..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/sax/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * Classes related to parsing and producing HTML from/to Boilerpipe TextDocuments.
- */
-package com.kohlschutter.boilerpipe.sax;
-
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/util/UnicodeTokenizer.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/util/UnicodeTokenizer.java
deleted file mode 100644
index 7bccdce9d1..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/util/UnicodeTokenizer.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * boilerpipe
- * <p>
- * Copyright (c) 2009, 2014 Christian Kohlschütter
- * <p>
- * The author licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kohlschutter.boilerpipe.util;
-
-import java.util.regex.Pattern;
-
-/**
- * Tokenizes text according to Unicode word boundaries and strips off non-word characters.
- */
-public class UnicodeTokenizer {
- private static final Pattern PAT_WORD_BOUNDARY = Pattern.compile("\\b");
- private static final Pattern PAT_NOT_WORD_BOUNDARY = Pattern
- .compile("[\u2063]*([\\\"'\\.,\\!\\@\\-\\:\\;\\$\\?\\(\\)/])[\u2063]*");
-
- /**
- * Tokenizes the text and returns an array of tokens.
- *
- * @param text The text
- * @return The tokens
- */
- public static String[] tokenize(final CharSequence text) {
- return PAT_NOT_WORD_BOUNDARY.matcher(PAT_WORD_BOUNDARY.matcher(text).replaceAll("\u2063"))
- .replaceAll("$1").replaceAll("[ \u2063]+", " ").trim().split("[ ]+");
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/util/package-info.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/util/package-info.java
deleted file mode 100644
index 844cd29a3d..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/com/kohlschutter/boilerpipe/util/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * Some helper classes.
- */
-package com.kohlschutter.boilerpipe.util;
-
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/AbstractFlinkTransformationProgram.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/AbstractFlinkTransformationProgram.java
deleted file mode 100644
index 3f058b4bce..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/AbstractFlinkTransformationProgram.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.pe.flink;
-
-import org.apache.streampipes.wrapper.flink.FlinkDataProcessorProgram;
-import org.apache.streampipes.wrapper.params.compat.ProcessorParams;
-
-public abstract class AbstractFlinkTransformationProgram<T extends ProcessorParams>
- extends FlinkDataProcessorProgram<T> {
-
- public AbstractFlinkTransformationProgram(T params) {
- super(params);
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/AbstractPatternDetectionProgram.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/AbstractPatternDetectionProgram.java
deleted file mode 100644
index 9835d9b15c..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/AbstractPatternDetectionProgram.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.pe.flink;
-
-import org.apache.streampipes.wrapper.flink.FlinkDataProcessorProgram;
-import org.apache.streampipes.wrapper.params.compat.ProcessorParams;
-
-import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-
-public abstract class AbstractPatternDetectionProgram<T extends ProcessorParams>
- extends FlinkDataProcessorProgram<T> {
-
- public AbstractPatternDetectionProgram(T params) {
- super(params);
- }
-
- @Override
- public void appendEnvironmentConfig(StreamExecutionEnvironment env) {
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setParallelism(1);
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/AllFlinkPipelineElementsInit.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/AllFlinkPipelineElementsInit.java
deleted file mode 100644
index 3344fa55b1..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/AllFlinkPipelineElementsInit.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.pe.flink;
-
-import org.apache.streampipes.extensions.management.model.SpServiceDefinition;
-import org.apache.streampipes.extensions.management.model.SpServiceDefinitionBuilder;
-import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory;
-import org.apache.streampipes.messaging.kafka.SpKafkaProtocolFactory;
-import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory;
-import org.apache.streampipes.messaging.nats.SpNatsProtocolFactory;
-import org.apache.streampipes.pe.flink.config.ConfigKeys;
-import org.apache.streampipes.pe.flink.processor.absence.AbsenceController;
-import org.apache.streampipes.pe.flink.processor.aggregation.AggregationController;
-import org.apache.streampipes.pe.flink.processor.and.AndController;
-import org.apache.streampipes.pe.flink.processor.boilerplate.BoilerplateController;
-import org.apache.streampipes.pe.flink.processor.converter.FieldConverterController;
-import org.apache.streampipes.pe.flink.processor.count.CountController;
-import org.apache.streampipes.pe.flink.processor.eventcount.EventCountController;
-import org.apache.streampipes.pe.flink.processor.gridenricher.SpatialGridEnrichmentController;
-import org.apache.streampipes.pe.flink.processor.measurementunitonverter.MeasurementUnitConverterController;
-import org.apache.streampipes.pe.flink.processor.peak.PeakDetectionController;
-import org.apache.streampipes.pe.flink.processor.rate.EventRateController;
-import org.apache.streampipes.pe.flink.processor.rename.FieldRenamerController;
-import org.apache.streampipes.pe.flink.processor.sequence.SequenceController;
-import org.apache.streampipes.pe.flink.processor.stat.summary.StatisticsSummaryController;
-import org.apache.streampipes.pe.flink.processor.stat.window.StatisticsSummaryControllerWindow;
-import org.apache.streampipes.pe.flink.processor.timestamp.TimestampController;
-import org.apache.streampipes.pe.flink.processor.urldereferencing.UrlDereferencingController;
-import org.apache.streampipes.pe.flink.processor.wordcount.WordCountController;
-import org.apache.streampipes.pe.flink.sink.elasticsearch.ElasticSearchController;
-import org.apache.streampipes.service.extensions.StreamPipesExtensionsServiceBase;
-import org.apache.streampipes.wrapper.flink.FlinkRuntimeProvider;
-
-
-public class AllFlinkPipelineElementsInit extends StreamPipesExtensionsServiceBase {
-
- public static void main(String[] args) {
- new AllFlinkPipelineElementsInit().init();
- }
-
- @Override
- public SpServiceDefinition provideServiceDefinition() {
- return SpServiceDefinitionBuilder.create("org-apache-streampipes-pe-all-flink",
- "StreamPipes Bundled Pipeline Elements for JVM Wrapper",
- "",
- 8090)
- .registerPipelineElements(new AggregationController(),
- new CountController(),
- new EventRateController(),
- new EventCountController(),
- new TimestampController(),
- new UrlDereferencingController(),
- new SpatialGridEnrichmentController(),
- new PeakDetectionController(),
- new SequenceController(),
- new AbsenceController(),
- new AndController(),
- new StatisticsSummaryController(),
- new StatisticsSummaryControllerWindow(),
- new WordCountController(),
- new FieldConverterController(),
- new MeasurementUnitConverterController(),
- new FieldRenamerController(),
- new BoilerplateController(),
- new ElasticSearchController())
- .registerRuntimeProvider(new FlinkRuntimeProvider())
- .registerMessagingProtocols(
- new SpKafkaProtocolFactory(),
- new SpJmsProtocolFactory(),
- new SpMqttProtocolFactory(),
- new SpNatsProtocolFactory())
- .addConfig(ConfigKeys.ELASTIC_HOST, "elasticsearch", "Elastic search host address")
- .addConfig(ConfigKeys.ELASTIC_PORT_REST, 9200, "Elasitc search rest port")
- .build();
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/config/ConfigKeys.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/config/ConfigKeys.java
deleted file mode 100644
index 994cf7b140..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/config/ConfigKeys.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.pe.flink.config;
-
-public class ConfigKeys {
- public static final String FLINK_HOST = "SP_FLINK_HOST";
- public static final String FLINK_PORT = "SP_FLINK_PORT";
- public static final String ELASTIC_HOST = "SP_ELASTICSEARCH_HOST";
- public static final String ELASTIC_PORT_REST = "SP_ELASTICSEARCH_PORT_REST";
- public static final String DEBUG = "SP_FLINK_DEBUG";
- public static final String FLINK_JAR_FILE_LOC = "SP_FLINK_JAR_FILE_LOC";
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/MapKeySelector.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/MapKeySelector.java
deleted file mode 100644
index 5538d90eb2..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/MapKeySelector.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.pe.flink.extensions;
-
-import org.apache.streampipes.model.runtime.Event;
-
-import org.apache.flink.api.java.functions.KeySelector;
-
-import java.io.Serializable;
-
-public class MapKeySelector implements Serializable {
-
- private String groupBy;
-
- public MapKeySelector(String groupBy) {
- this.groupBy = groupBy;
- }
-
- public KeySelector<Event, String> getKeySelector() {
- return new KeySelector<Event, String>() {
- @Override
- public String getKey(Event in) throws Exception {
- return in.getFieldBySelector(groupBy).getAsPrimitive().getAsString();
- }
- };
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/SlidingBatchWindow.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/SlidingBatchWindow.java
deleted file mode 100644
index 9dc043196c..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/SlidingBatchWindow.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.pe.flink.extensions;
-
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
-import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class SlidingBatchWindow<T> extends AbstractStreamOperator<List<T>> implements
- OneInputStreamOperator<T, List<T>> {
-
- private Integer windowSize;
- private List<T> currentEvents;
-
- public SlidingBatchWindow(Integer windowSize) {
- super();
- this.windowSize = windowSize;
- this.currentEvents = new ArrayList<>();
- }
-
- @Override
- public void processElement(StreamRecord<T> in) throws Exception {
- currentEvents.add(in.getValue());
- if (currentEvents.size() > windowSize) {
- currentEvents.remove(0);
- }
-
- output.collect(new StreamRecord<>(currentEvents, System.currentTimeMillis()));
-
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/SlidingEventTimeWindow.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/SlidingEventTimeWindow.java
deleted file mode 100644
index 94a1aefd93..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/SlidingEventTimeWindow.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.pe.flink.extensions;
-
-import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
-import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-public class SlidingEventTimeWindow<T> extends AbstractUdfStreamOperator<List<T>,
- TimestampMappingFunction<T>>
- implements
- OneInputStreamOperator<T, List<T>>, Serializable {
-
- private Long timeWindowSizeInMillis;
- private List<T> currentEvents;
-
- public SlidingEventTimeWindow(Long time, TimeUnit timeUnit, TimestampMappingFunction<T>
- timestampMappingFunction) {
- super(timestampMappingFunction);
- this.timeWindowSizeInMillis = toMilliseconds(time, timeUnit);
- this.currentEvents = new ArrayList<>();
-
- }
-
- private Long toMilliseconds(Long time, TimeUnit timeUnit) {
- return timeUnit.toMillis(time);
- }
-
- @Override
- public void processElement(StreamRecord<T> in) throws Exception {
- Long currentTimestamp = userFunction.getTimestamp(in.getValue());
-
- checkForRemoval(currentTimestamp);
- currentEvents.add(in.getValue());
-
- output.collect(new StreamRecord<>(currentEvents, System.currentTimeMillis()));
-
- }
-
- private void checkForRemoval(Long currentTimestamp) {
- Iterator<T> it = currentEvents.iterator();
-
- while (it.hasNext()) {
- T next = it.next();
- if (removalRequired(userFunction.getTimestamp(next), currentTimestamp)) {
- it.remove();
- } else {
- break;
- }
- }
- }
-
- private boolean removalRequired(Long oldTimestamp, Long currentTimestamp) {
- return (currentTimestamp - oldTimestamp > timeWindowSizeInMillis);
- }
-
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/TimestampMappingFunction.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/TimestampMappingFunction.java
deleted file mode 100644
index 1f9d96cc25..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/extensions/TimestampMappingFunction.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.pe.flink.extensions;
-
-import org.apache.flink.api.common.functions.Function;
-
-import java.io.Serializable;
-
-public interface TimestampMappingFunction<T> extends Function, Serializable {
-
- Long getTimestamp(T in);
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/processor/absence/Absence.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/processor/absence/Absence.java
deleted file mode 100644
index 8a00c7a5e6..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/processor/absence/Absence.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.pe.flink.processor.absence;
-
-public class Absence {
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/processor/absence/AbsenceController.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/processor/absence/AbsenceController.java
deleted file mode 100644
index 68abf427fd..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/processor/absence/AbsenceController.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.pe.flink.processor.absence;
-
-import org.apache.streampipes.model.DataProcessorType;
-import org.apache.streampipes.model.graph.DataProcessorDescription;
-import org.apache.streampipes.model.graph.DataProcessorInvocation;
-import org.apache.streampipes.model.schema.EventProperty;
-import org.apache.streampipes.pe.flink.processor.and.TimeUnit;
-import org.apache.streampipes.sdk.builder.ProcessingElementBuilder;
-import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
-import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
-import org.apache.streampipes.sdk.helpers.EpRequirements;
-import org.apache.streampipes.sdk.helpers.Labels;
-import org.apache.streampipes.sdk.helpers.Locales;
-import org.apache.streampipes.sdk.helpers.Options;
-import org.apache.streampipes.sdk.helpers.OutputStrategies;
-import org.apache.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
-import org.apache.streampipes.wrapper.flink.FlinkDataProcessorProgram;
-import org.apache.streampipes.model.extensions.ExtensionAssetType;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class AbsenceController extends FlinkDataProcessorDeclarer<AbsenceParameters> {
-
- private static final String TIME_WINDOW = "time-window";
- private static final String TIME_UNIT = "time-unit";
-
- @Override
- public DataProcessorDescription declareModel() {
- return ProcessingElementBuilder.create("org.apache.streampipes.processors.pattern-detection.flink.absence")
- .withLocales(Locales.EN)
- .withAssets(ExtensionAssetType.DOCUMENTATION)
- .category(DataProcessorType.PATTERN_DETECT)
- .requiredStream(StreamRequirementsBuilder
- .create()
- .requiredProperty(EpRequirements.anyProperty())
- .build())
- .requiredStream(StreamRequirementsBuilder
- .create()
- .requiredProperty(EpRequirements.anyProperty())
- .build())
- .requiredSingleValueSelection(Labels.withId(TIME_UNIT), Options.from("Seconds",
- "Minutes", "Hours"))
- .requiredIntegerParameter(Labels.withId(TIME_WINDOW))
- .outputStrategy(OutputStrategies.custom(false))
- .build();
- }
-
- @Override
- public FlinkDataProcessorProgram<AbsenceParameters> getProgram(DataProcessorInvocation graph,
- ProcessingElementParameterExtractor extractor) {
- List<String> selectProperties = new ArrayList<>();
- for (EventProperty p : graph.getOutputStream().getEventSchema().getEventProperties()) {
- selectProperties.add(p.getRuntimeName());
- }
-
- TimeUnit timeUnit = TimeUnit.valueOf(extractor.selectedSingleValue(TIME_UNIT, String.class));
- Integer timeWindow = extractor.singleValueParameter(TIME_WINDOW, Integer.class);
-
- AbsenceParameters params = new AbsenceParameters(graph, selectProperties, timeWindow, timeUnit);
-
- return new AbsenceProgram(params);
- }
-}
diff --git a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/processor/absence/AbsenceParameters.java b/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/processor/absence/AbsenceParameters.java
deleted file mode 100644
index d5f42c3e38..0000000000
--- a/streampipes-extensions/streampipes-pipeline-elements-experimental-flink/src/main/java/org/apache/streampipes/pe/flink/processor/absence/AbsenceParameters.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.pe.flink.processor.absence;
-
-import org.apache.streampipes.model.graph.DataProcessorInvocation;
-import org.apache.streampipes.pe.flink.processor.and.TimeUnit;
-import org.apache.streampipes.wrapper.params.compat.ProcessorParams;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class AbsenceParameters extends ProcessorParams {
-
- private static final long serialVersionUID = 4319341875274736697L;
-
- private List<String> selectProperties = new ArrayList<>();
- private Integer timeWindowSize;
- private TimeUnit timeUnit;
-
- public AbsenceParameters(DataProcessorInvocation graph, List<String> selectProperties, Integer timeWindowSize,
- TimeUnit timeUnit) {
- super(graph);
- this.selectProperties = selectProperties;
- this.timeWindowSize = timeWindowSize;
- this.timeUnit = timeUnit;
- }
-
... 11425 lines suppressed ...