You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2021/12/15 21:05:29 UTC
[incubator-streampipes] 01/02: [STREAMPIPES-484] Use client API for file requests in CSV Metadata Enricher, deprecate SDK methods
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 8864cf416bafc201361be9c3d78549d5c69f6826
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Wed Dec 15 22:00:42 2021 +0100
[STREAMPIPES-484] Use client API for file requests in CSV Metadata Enricher, deprecate SDK methods
---
.../org/apache/streampipes/client/api/FileApi.java | 8 ++++--
.../CsvMetadataEnrichmentController.java | 32 ++++++++++++++--------
.../sdk/extractor/AbstractParameterExtractor.java | 25 +++++++++++++----
3 files changed, 46 insertions(+), 19 deletions(-)
diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java
index aebf041..256ea18 100644
--- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java
+++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java
@@ -29,8 +29,12 @@ public class FileApi extends AbstractClientApi {
super(clientConfig);
}
- public byte[] getFileContent(String fileName) {
- return new BinaryGetRequest(clientConfig, getBaseResourcePath(fileName), null).executeRequest();
+ public byte[] getFileContent(String filename) {
+ return new BinaryGetRequest(clientConfig, getBaseResourcePath(filename), null).executeRequest();
+ }
+
+ public String getFileContentAsString(String filename) {
+ return new String(getFileContent(filename));
}
public void writeToFile(String file, String fileLocation) {
diff --git a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentController.java b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentController.java
index a0ea2ba..8aa0c6e 100644
--- a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentController.java
+++ b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentController.java
@@ -19,6 +19,7 @@ package org.apache.streampipes.processors.transformation.jvm.processor.csvmetada
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
+import org.apache.streampipes.client.StreamPipesClient;
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.container.api.ResolvesContainerProvidedOptions;
import org.apache.streampipes.container.api.ResolvesContainerProvidedOutputStrategy;
@@ -30,10 +31,12 @@ import org.apache.streampipes.model.schema.PropertyScope;
import org.apache.streampipes.model.staticproperty.Option;
import org.apache.streampipes.sdk.builder.ProcessingElementBuilder;
import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.apache.streampipes.sdk.extractor.AbstractParameterExtractor;
import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
import org.apache.streampipes.sdk.extractor.StaticPropertyExtractor;
import org.apache.streampipes.sdk.helpers.*;
import org.apache.streampipes.sdk.utils.Assets;
+import org.apache.streampipes.service.extensions.base.client.StreamPipesClientResolver;
import org.apache.streampipes.wrapper.standalone.ConfiguredEventProcessor;
import org.apache.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
@@ -78,16 +81,12 @@ public class CsvMetadataEnrichmentController
}
@Override
- public ConfiguredEventProcessor<CsvMetadataEnrichmentParameters> onInvocation(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ public ConfiguredEventProcessor<CsvMetadataEnrichmentParameters> onInvocation(DataProcessorInvocation graph,
+ ProcessingElementParameterExtractor extractor) {
String mappingFieldSelector = extractor.mappingPropertyValue(MAPPING_FIELD_KEY);
List<String> fieldsToAppend = extractor.selectedMultiValues(FIELDS_TO_APPEND_KEY, String.class);
String lookupField = extractor.selectedSingleValue(FIELD_TO_MATCH, String.class);
- String fileContents = null;
- try {
- fileContents = extractor.fileContentsAsString(CSV_FILE_KEY);
- } catch (IOException e) {
- e.printStackTrace();
- }
+ String fileContents = getFileContents(extractor);
CsvMetadataEnrichmentParameters params = new CsvMetadataEnrichmentParameters(graph,
mappingFieldSelector,
@@ -99,9 +98,10 @@ public class CsvMetadataEnrichmentController
}
@Override
- public List<Option> resolveOptions(String requestId, StaticPropertyExtractor parameterExtractor) {
+ public List<Option> resolveOptions(String requestId,
+ StaticPropertyExtractor parameterExtractor) {
try {
- String fileContents = parameterExtractor.fileContentsAsString(CSV_FILE_KEY);
+ String fileContents = getFileContents(parameterExtractor);
if (requestId.equals(FIELDS_TO_APPEND_KEY)) {
String matchColumn = parameterExtractor.selectedSingleValue(FIELD_TO_MATCH, String.class);
return getOptionsFromColumnNames(fileContents, Collections.singletonList(matchColumn));
@@ -115,7 +115,8 @@ public class CsvMetadataEnrichmentController
}
@Override
- public EventSchema resolveOutputStrategy(DataProcessorInvocation processingElement, ProcessingElementParameterExtractor parameterExtractor) throws SpRuntimeException {
+ public EventSchema resolveOutputStrategy(DataProcessorInvocation processingElement,
+ ProcessingElementParameterExtractor parameterExtractor) throws SpRuntimeException {
List<EventProperty> properties = processingElement
.getInputStreams()
.get(0)
@@ -125,7 +126,7 @@ public class CsvMetadataEnrichmentController
List<String> columnsToInclude = parameterExtractor.selectedMultiValues(FIELDS_TO_APPEND_KEY,
String.class);
try {
- String fileContents = parameterExtractor.fileContentsAsString(CSV_FILE_KEY);
+ String fileContents = getFileContents(parameterExtractor);
properties.addAll(getAppendProperties(fileContents, columnsToInclude));
} catch (IOException e) {
e.printStackTrace();
@@ -169,4 +170,13 @@ public class CsvMetadataEnrichmentController
.filter(key -> columnsToIgnore.stream().noneMatch(c -> c.equals(key)))
.collect(Collectors.toList());
}
+
+ private String getFileContents(AbstractParameterExtractor<?> extractor) {
+ String filename = extractor.selectedFilename(CSV_FILE_KEY);
+ return getStreamPipesClientInstance().fileApi().getFileContentAsString(filename);
+ }
+
+ private StreamPipesClient getStreamPipesClientInstance() {
+ return new StreamPipesClientResolver().makeStreamPipesClientInstance();
+ }
}
diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java
index 6c698d3..bdd17a6 100644
--- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java
+++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java
@@ -108,19 +108,32 @@ public abstract class AbstractParameterExtractor<T extends InvocableStreamPipesE
return getStaticPropertyByName(internalName, ColorPickerStaticProperty.class).getSelectedColor();
}
+ /**
+ * @deprecated
+ * This won't work after release 0.69.0 as all API requests against the core need to be authenticated.
+ * Use the StreamPipes Client File API instead.
+ **/
+ @Deprecated
public String fileContentsAsString(String internalName) throws IOException {
- String filename = selectedFilename(internalName);
- return Request.Get(makeFileRequestPath(filename)).execute().returnContent().asString(Charsets.UTF_8);
+ throw new IllegalArgumentException("Deprecated as API requests need to be authenticated - use the StreamPipes Client file API instead.");
}
+ /**
+ * @deprecated
+ * This won't work after release 0.69.0 as all API requests against the core need to be authenticated.
+ * Use the StreamPipes Client File API instead.
+ **/
public byte[] fileContentsAsByteArray(String internalName) throws IOException {
- String filename = selectedFilename(internalName);
- return Request.Get(makeFileRequestPath(filename)).execute().returnContent().asBytes();
+ throw new IllegalArgumentException("Deprecated as API requests need to be authenticated - use the StreamPipes Client file API instead.");
}
+ /**
+ * @deprecated
+ * This won't work after release 0.69.0 as all API requests against the core need to be authenticated.
+ * Use the StreamPipes Client File API instead.
+ **/
public InputStream fileContentsAsStream(String internalName) throws IOException {
- String filename = selectedFilename(internalName);
- return Request.Get(makeFileRequestPath(filename)).execute().returnContent().asStream();
+ throw new IllegalArgumentException("Deprecated as API requests need to be authenticated - use the StreamPipes Client file API instead.");
}
public String selectedFilename(String internalName) {