You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by tu...@apache.org on 2023/02/20 21:14:38 UTC
[nifi] branch main updated: NIFI-11190: Added provenance events to GetHubSpot and GetShopify
This is an automated email from the ASF dual-hosted git repository.
turcsanyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 1b951818e1 NIFI-11190: Added provenance events to GetHubSpot and GetShopify
1b951818e1 is described below
commit 1b951818e1fbf7f09113e93811008627a17f2ed0
Author: Lehel <Le...@hotmail.com>
AuthorDate: Thu Feb 16 17:28:19 2023 +0100
NIFI-11190: Added provenance events to GetHubSpot and GetShopify
This closes #6965.
Signed-off-by: Peter Turcsanyi <tu...@apache.org>
---
.../java/org/apache/nifi/processors/hubspot/GetHubSpot.java | 1 +
.../org/apache/nifi/processors/hubspot/GetHubSpotTest.java | 9 +++++++++
.../java/org/apache/nifi/processors/shopify/GetShopify.java | 3 +++
.../nifi/processors/shopify/rest/ShopifyRestService.java | 4 ++++
.../shopify/{GetShopifyIT.java => GetShopifyTest.java} | 12 ++++++++----
5 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/main/java/org/apache/nifi/processors/hubspot/GetHubSpot.java b/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/main/java/org/apache/nifi/processors/hubspot/GetHubSpot.java
index 53567ff212..d624b58afe 100644
--- a/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/main/java/org/apache/nifi/processors/hubspot/GetHubSpot.java
+++ b/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/main/java/org/apache/nifi/processors/hubspot/GetHubSpot.java
@@ -239,6 +239,7 @@ public class GetHubSpot extends AbstractProcessor {
if (total.get() > 0) {
flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/json");
session.transfer(flowFile, REL_SUCCESS);
+ session.getProvenanceReporter().receive(flowFile, uri.toString());
} else {
getLogger().debug("Empty response when requested HubSpot endpoint: [{}]", endpoint);
context.yield();
diff --git a/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/test/java/org/apache/nifi/processors/hubspot/GetHubSpotTest.java b/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/test/java/org/apache/nifi/processors/hubspot/GetHubSpotTest.java
index fd8ed18fed..55f371acf6 100644
--- a/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/test/java/org/apache/nifi/processors/hubspot/GetHubSpotTest.java
+++ b/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/test/java/org/apache/nifi/processors/hubspot/GetHubSpotTest.java
@@ -45,6 +45,7 @@ import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.state.Scope;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
@@ -109,6 +110,8 @@ class GetHubSpotTest {
flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/json");
assertEquals(expectedJsonNode, actualJsonNode);
+ List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri());
}
@Test
@@ -122,6 +125,7 @@ class GetHubSpotTest {
final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetHubSpot.REL_SUCCESS);
assertTrue(flowFiles.isEmpty());
+ assertTrue(runner.getProvenanceEvents().isEmpty());
}
@Test
@@ -131,6 +135,7 @@ class GetHubSpotTest {
server.enqueue(new MockResponse().setBody(response).setResponseCode(429));
assertThrows(AssertionError.class, () -> runner.run(1));
+ assertTrue(runner.getProvenanceEvents().isEmpty());
}
@Test
@@ -173,6 +178,8 @@ class GetHubSpotTest {
final String expectedJsonString = root.toString();
assertEquals(OBJECT_MAPPER.readTree(expectedJsonString), OBJECT_MAPPER.readTree(requestBodyString));
+ List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri());
}
@Test
@@ -220,6 +227,8 @@ class GetHubSpotTest {
final String expectedJsonString = root.toString();
assertEquals(OBJECT_MAPPER.readTree(expectedJsonString), OBJECT_MAPPER.readTree(requestBodyString));
+ List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri());
}
static class MockGetHubSpot extends GetHubSpot {
diff --git a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/GetShopify.java b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/GetShopify.java
index 9994ab999a..f0aee1514d 100644
--- a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/GetShopify.java
+++ b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/GetShopify.java
@@ -327,6 +327,7 @@ public class GetShopify extends AbstractProcessor {
if (objectCountHolder.get() > 0) {
flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/json");
session.transfer(flowFile, REL_SUCCESS);
+ session.getProvenanceReporter().receive(flowFile, shopifyRestService.getBaseUriString());
} else {
getLogger().debug("Empty response when requested Shopify resource: [{}]", resourceName);
session.remove(flowFile);
@@ -415,4 +416,6 @@ public class GetShopify extends AbstractProcessor {
throw new ProcessException("Clearing state failed", e);
}
}
+
+
}
diff --git a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/rest/ShopifyRestService.java b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/rest/ShopifyRestService.java
index 5bc7ad2f88..f9a8fc1b6a 100644
--- a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/rest/ShopifyRestService.java
+++ b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/rest/ShopifyRestService.java
@@ -76,6 +76,10 @@ public class ShopifyRestService {
return retrieveResponse(uri);
}
+ public String getBaseUriString() {
+ return getBaseUri().build().toString();
+ }
+
private HttpResponseEntity retrieveResponse(URI uri) {
return webClientServiceProvider.getWebClientService()
.get()
diff --git a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyIT.java b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyTest.java
similarity index 92%
rename from nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyIT.java
rename to nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyTest.java
index 73ed2a916b..a6f526fe82 100644
--- a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyIT.java
+++ b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyTest.java
@@ -24,6 +24,7 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processors.shopify.model.IncrementalLoadingParameter;
import org.apache.nifi.processors.shopify.model.ResourceType;
import org.apache.nifi.processors.shopify.rest.ShopifyRestService;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.MockProcessContext;
@@ -39,7 +40,6 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.time.Instant;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -49,7 +49,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
-class GetShopifyIT {
+class GetShopifyTest {
private static final String BASE_URL = "/test/shopify";
private static MockWebServer server;
@@ -101,6 +101,8 @@ class GetShopifyIT {
List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetShopify.REL_SUCCESS);
flowFiles.get(0).assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/json");
+ List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri());
}
@Test
@@ -124,6 +126,7 @@ class GetShopifyIT {
assertThrows(AssertionError.class, () -> runner.run(1));
assertTrue(((MockProcessContext) runner.getProcessContext()).isYieldCalled(),
"In case of 429 the processor should be yielded.");
+ assertTrue(runner.getProvenanceEvents().isEmpty());
}
@Test
@@ -148,6 +151,7 @@ class GetShopifyIT {
assertTrue(((MockProcessContext) runner.getProcessContext()).isYieldCalled(),
"In case of 404 the processor should be yielded.");
+ assertTrue(runner.getProvenanceEvents().isEmpty());
}
@Test
@@ -157,8 +161,6 @@ class GetShopifyIT {
.setBody(getResourceAsString("collection_listings.json"));
server.enqueue(mockResponse);
- final Instant expectedExecutionTime = Instant.parse("2022-08-16T10:15:30Z");
-
final StandardWebClientServiceProvider standardWebClientServiceProvider =
new StandardWebClientServiceProvider();
final CustomGetShopify customGetShopify = new CustomGetShopify();
@@ -177,6 +179,8 @@ class GetShopifyIT {
List<MockFlowFile> flowFilesForRelationship = runner.getFlowFilesForRelationship(GetShopify.REL_SUCCESS);
assertEquals(1, flowFilesForRelationship.size());
+ List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
+ assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri());
}
static class CustomGetShopify extends GetShopify {