You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2021/12/15 21:05:30 UTC
[incubator-streampipes] 02/02: [STREAMPIPES-484] Use client API for file requests in text mining components
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 380f0d8ab580a909fbba957cbae35ec684f96144
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Wed Dec 15 22:05:18 2021 +0100
[STREAMPIPES-484] Use client API for file requests in text mining components
---
.../textmining/jvm/processor/chunker/ChunkerController.java | 13 ++++++-------
.../jvm/processor/language/LanguageDetectionController.java | 12 +++++-------
.../jvm/processor/namefinder/NameFinderController.java | 12 +++++-------
.../jvm/processor/partofspeech/PartOfSpeechController.java | 12 +++++-------
.../sentencedetection/SentenceDetectionController.java | 12 +++++-------
.../jvm/processor/tokenizer/TokenizerController.java | 12 +++++-------
.../sdk/extractor/AbstractParameterExtractor.java | 6 +++---
7 files changed, 34 insertions(+), 45 deletions(-)
diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerController.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerController.java
index 1fe7c92..284912a 100644
--- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerController.java
+++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerController.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.textmining.jvm.processor.chunker;
+import org.apache.streampipes.client.StreamPipesClient;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -28,6 +29,7 @@ import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
import org.apache.streampipes.sdk.helpers.*;
import org.apache.streampipes.sdk.utils.Assets;
import org.apache.streampipes.sdk.utils.Datatypes;
+import org.apache.streampipes.service.extensions.base.client.StreamPipesClientResolver;
import org.apache.streampipes.wrapper.standalone.ConfiguredEventProcessor;
import org.apache.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
@@ -74,16 +76,13 @@ public class ChunkerController extends StandaloneEventProcessingDeclarer<Chunker
@Override
public ConfiguredEventProcessor<ChunkerParameters> onInvocation(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance();
+ String filename = extractor.selectedFilename(BINARY_FILE_KEY);
+ byte[] fileContent = client.fileApi().getFileContent(filename);
+
String tags = extractor.mappingPropertyValue(TAGS_FIELD_KEY);
String tokens = extractor.mappingPropertyValue(TOKENS_FIELD_KEY);
- byte[] fileContent = null;
- try {
- fileContent = extractor.fileContentsAsByteArray(BINARY_FILE_KEY);
- } catch (IOException e) {
- e.printStackTrace();
- }
-
ChunkerParameters params = new ChunkerParameters(graph, tags, tokens, fileContent);
return new ConfiguredEventProcessor<>(params, Chunker::new);
}
diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionController.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionController.java
index be3cf0b..e0555d9 100644
--- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionController.java
+++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionController.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.textmining.jvm.processor.language;
+import org.apache.streampipes.client.StreamPipesClient;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -27,6 +28,7 @@ import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
import org.apache.streampipes.sdk.helpers.*;
import org.apache.streampipes.sdk.utils.Assets;
+import org.apache.streampipes.service.extensions.base.client.StreamPipesClientResolver;
import org.apache.streampipes.wrapper.standalone.ConfiguredEventProcessor;
import org.apache.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
@@ -68,15 +70,11 @@ public class LanguageDetectionController extends StandaloneEventProcessingDeclar
@Override
public ConfiguredEventProcessor<LanguageDetectionParameters> onInvocation(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance();
+ String filename = extractor.selectedFilename(BINARY_FILE_KEY);
+ byte[] fileContent = client.fileApi().getFileContent(filename);
String detection = extractor.mappingPropertyValue(DETECTION_FIELD_KEY);
- byte[] fileContent = null;
- try {
- fileContent = extractor.fileContentsAsByteArray(BINARY_FILE_KEY);
- } catch (IOException e) {
- e.printStackTrace();
- }
-
LanguageDetectionParameters params = new LanguageDetectionParameters(graph, detection, fileContent);
return new ConfiguredEventProcessor<>(params, LanguageDetection::new);
}
diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderController.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderController.java
index 195d75c..c32092b 100644
--- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderController.java
+++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderController.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.textmining.jvm.processor.namefinder;
+import org.apache.streampipes.client.StreamPipesClient;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -28,6 +29,7 @@ import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
import org.apache.streampipes.sdk.helpers.*;
import org.apache.streampipes.sdk.utils.Assets;
import org.apache.streampipes.sdk.utils.Datatypes;
+import org.apache.streampipes.service.extensions.base.client.StreamPipesClientResolver;
import org.apache.streampipes.wrapper.standalone.ConfiguredEventProcessor;
import org.apache.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
@@ -64,15 +66,11 @@ public class NameFinderController extends StandaloneEventProcessingDeclarer<Name
@Override
public ConfiguredEventProcessor<NameFinderParameters> onInvocation(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance();
+ String filename = extractor.selectedFilename(MODEL);
+ byte[] fileContent = client.fileApi().getFileContent(filename);
String tokens = extractor.mappingPropertyValue(TOKENS_FIELD_KEY);
- byte[] fileContent = null;
- try {
- fileContent = extractor.fileContentsAsByteArray(MODEL);
- } catch (IOException e) {
- e.printStackTrace();
- }
-
NameFinderParameters params = new NameFinderParameters(graph, tokens, fileContent);
return new ConfiguredEventProcessor<>(params, NameFinder::new);
}
diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechController.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechController.java
index d2ea1c9..c40659a 100644
--- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechController.java
+++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechController.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.textmining.jvm.processor.partofspeech;
+import org.apache.streampipes.client.StreamPipesClient;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -28,6 +29,7 @@ import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
import org.apache.streampipes.sdk.helpers.*;
import org.apache.streampipes.sdk.utils.Assets;
import org.apache.streampipes.sdk.utils.Datatypes;
+import org.apache.streampipes.service.extensions.base.client.StreamPipesClientResolver;
import org.apache.streampipes.wrapper.standalone.ConfiguredEventProcessor;
import org.apache.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
@@ -69,15 +71,11 @@ public class PartOfSpeechController extends StandaloneEventProcessingDeclarer<Pa
@Override
public ConfiguredEventProcessor<PartOfSpeechParameters> onInvocation(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance();
+ String filename = extractor.selectedFilename(BINARY_FILE_KEY);
+ byte[] fileContent = client.fileApi().getFileContent(filename);
String detection = extractor.mappingPropertyValue(DETECTION_FIELD_KEY);
- byte[] fileContent = null;
- try {
- fileContent = extractor.fileContentsAsByteArray(BINARY_FILE_KEY);
- } catch (IOException e) {
- e.printStackTrace();
- }
-
PartOfSpeechParameters params = new PartOfSpeechParameters(graph, detection, fileContent);
return new ConfiguredEventProcessor<>(params, PartOfSpeech::new);
}
diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionController.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionController.java
index 6516bd0..b938001 100644
--- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionController.java
+++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionController.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.textmining.jvm.processor.sentencedetection;
+import org.apache.streampipes.client.StreamPipesClient;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -27,6 +28,7 @@ import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
import org.apache.streampipes.sdk.helpers.*;
import org.apache.streampipes.sdk.utils.Assets;
+import org.apache.streampipes.service.extensions.base.client.StreamPipesClientResolver;
import org.apache.streampipes.wrapper.standalone.ConfiguredEventProcessor;
import org.apache.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
@@ -58,15 +60,11 @@ public class SentenceDetectionController extends StandaloneEventProcessingDeclar
@Override
public ConfiguredEventProcessor<SentenceDetectionParameters> onInvocation(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance();
+ String filename = extractor.selectedFilename(BINARY_FILE_KEY);
+ byte[] fileContent = client.fileApi().getFileContent(filename);
String detection = extractor.mappingPropertyValue(DETECTION_FIELD_KEY);
- byte[] fileContent = null;
- try {
- fileContent = extractor.fileContentsAsByteArray(BINARY_FILE_KEY);
- } catch (IOException e) {
- e.printStackTrace();
- }
-
SentenceDetectionParameters params = new SentenceDetectionParameters(graph, detection, fileContent);
return new ConfiguredEventProcessor<>(params, SentenceDetection::new);
}
diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerController.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerController.java
index dececa2..7914d00 100644
--- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerController.java
+++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerController.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.processors.textmining.jvm.processor.tokenizer;
+import org.apache.streampipes.client.StreamPipesClient;
import org.apache.streampipes.model.DataProcessorType;
import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
@@ -27,6 +28,7 @@ import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
import org.apache.streampipes.sdk.helpers.*;
import org.apache.streampipes.sdk.utils.Assets;
+import org.apache.streampipes.service.extensions.base.client.StreamPipesClientResolver;
import org.apache.streampipes.wrapper.standalone.ConfiguredEventProcessor;
import org.apache.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
@@ -62,15 +64,11 @@ public class TokenizerController extends StandaloneEventProcessingDeclarer<Token
@Override
public ConfiguredEventProcessor<TokenizerParameters> onInvocation(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance();
+ String filename = extractor.selectedFilename(BINARY_FILE_KEY);
+ byte[] fileContent = client.fileApi().getFileContent(filename);
String detection = extractor.mappingPropertyValue(DETECTION_FIELD_KEY);
- byte[] fileContent = null;
- try {
- fileContent = extractor.fileContentsAsByteArray(BINARY_FILE_KEY);
- } catch (IOException e) {
- e.printStackTrace();
- }
-
TokenizerParameters params = new TokenizerParameters(graph, detection, fileContent);
return new ConfiguredEventProcessor<>(params, Tokenizer::new);
}
diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java
index bdd17a6..6802c32 100644
--- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java
+++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java
@@ -111,7 +111,7 @@ public abstract class AbstractParameterExtractor<T extends InvocableStreamPipesE
/**
* @deprecated
* This won't work after release 0.69.0 as all API requests against the core need to be authenticated.
- * Use the StreamPipes Client File API instead.
+ * Use the StreamPipes Client File API instead (e.g., StreamPipesClientResolver.makeStreamPipesClientInstance()).
**/
@Deprecated
public String fileContentsAsString(String internalName) throws IOException {
@@ -121,7 +121,7 @@ public abstract class AbstractParameterExtractor<T extends InvocableStreamPipesE
/**
* @deprecated
* This won't work after release 0.69.0 as all API requests against the core need to be authenticated.
- * Use the StreamPipes Client File API instead.
+ * Use the StreamPipes Client File API instead (e.g., StreamPipesClientResolver.makeStreamPipesClientInstance()).
**/
public byte[] fileContentsAsByteArray(String internalName) throws IOException {
throw new IllegalArgumentException("Deprecated as API requests need to be authenticated - use the StreamPipes Client file API instead.");
@@ -130,7 +130,7 @@ public abstract class AbstractParameterExtractor<T extends InvocableStreamPipesE
/**
* @deprecated
* This won't work after release 0.69.0 as all API requests against the core need to be authenticated.
- * Use the StreamPipes Client File API instead.
+ * Use the StreamPipes Client File API instead (e.g., StreamPipesClientResolver.makeStreamPipesClientInstance()).
**/
public InputStream fileContentsAsStream(String internalName) throws IOException {
throw new IllegalArgumentException("Deprecated as API requests need to be authenticated - use the StreamPipes Client file API instead.");