You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ex...@apache.org on 2024/07/24 23:51:56 UTC
(nifi) branch main updated: NIFI-13581 Remove Parameter Contexts and Providers in destroyFlow of NiFiSystemIT (#9110)
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 94f8688545 NIFI-13581 Remove Parameter Contexts and Providers in destroyFlow of NiFiSystemIT (#9110)
94f8688545 is described below
commit 94f8688545873fd05b8dd8cf7d962fbc902b184d
Author: Bryan Bende <bb...@apache.org>
AuthorDate: Wed Jul 24 19:51:49 2024 -0400
NIFI-13581 Remove Parameter Contexts and Providers in destroyFlow of NiFiSystemIT (#9110)
Signed-off-by: David Handermann <ex...@apache.org>
---
.github/workflows/system-tests.yml | 4 +--
.../apache/nifi/tests/system/NiFiClientUtil.java | 36 ++++++++++++++++++++++
.../org/apache/nifi/tests/system/NiFiSystemIT.java | 2 ++
.../cli/impl/client/nifi/ParamContextClient.java | 2 ++
.../cli/impl/client/nifi/ParamProviderClient.java | 2 ++
.../client/nifi/impl/JerseyParamContextClient.java | 12 +++++++-
.../nifi/impl/JerseyParamProviderClient.java | 12 +++++++-
7 files changed, 66 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 9e083e7c54..db2c337628 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -26,7 +26,7 @@ on:
- 'nifi-system-tests/**'
- 'nifi-api/**'
- 'nifi-framework-api/**'
- - 'nifi-nifi-framework-bundle/**'
+ - 'nifi-framework-bundle/**'
- 'nifi-extension-bundles/nifi-py4j-bundle/**'
- 'nifi-stateless/**'
pull_request:
@@ -36,7 +36,7 @@ on:
- 'nifi-system-tests/**'
- 'nifi-api/**'
- 'nifi-framework-api/**'
- - 'nifi-nifi-framework-bundle/**'
+ - 'nifi-framework-bundle/**'
- 'nifi-extension-bundles/nifi-py4j-bundle/**'
- 'nifi-stateless/**'
diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java
index 8e62196f6e..f9aa09a396 100644
--- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java
+++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java
@@ -89,6 +89,7 @@ import org.apache.nifi.web.api.entity.NodeEntity;
import org.apache.nifi.web.api.entity.ParameterContextEntity;
import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity;
import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity;
+import org.apache.nifi.web.api.entity.ParameterContextsEntity;
import org.apache.nifi.web.api.entity.ParameterEntity;
import org.apache.nifi.web.api.entity.ParameterGroupConfigurationEntity;
import org.apache.nifi.web.api.entity.ParameterProviderApplyParametersRequestEntity;
@@ -96,6 +97,7 @@ import org.apache.nifi.web.api.entity.ParameterProviderConfigurationEntity;
import org.apache.nifi.web.api.entity.ParameterProviderEntity;
import org.apache.nifi.web.api.entity.ParameterProviderParameterApplicationEntity;
import org.apache.nifi.web.api.entity.ParameterProviderParameterFetchEntity;
+import org.apache.nifi.web.api.entity.ParameterProvidersEntity;
import org.apache.nifi.web.api.entity.PortEntity;
import org.apache.nifi.web.api.entity.ProcessGroupEntity;
import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity;
@@ -126,11 +128,13 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import java.util.stream.Collectors;
public class NiFiClientUtil {
@@ -472,6 +476,38 @@ public class NiFiClientUtil {
}
}
+ public void deleteParameterContexts() throws NiFiClientException, IOException {
+ final ParameterContextsEntity parameterContextsEntity = nifiClient.getParamContextClient().getParamContexts();
+ final Map<String, ParameterContextEntity> parameterContextMap = parameterContextsEntity.getParameterContexts().stream()
+ .collect(Collectors.toMap(ParameterContextEntity::getId, Function.identity()));
+
+ // If parameter context have inherited contexts then they needed to be deleted from the bottom up, so we just keep iterating
+ // over the set of ids and retrying deletes until all have been deleted, knowing that some delete calls will fail the first time
+ final Set<String> parameterContextIds = new HashSet<>(parameterContextMap.keySet());
+ while (!parameterContextIds.isEmpty()) {
+ final Iterator<String> parameterContextIdIterator = parameterContextIds.iterator();
+ while (parameterContextIdIterator.hasNext()) {
+ final String parameterContextId = parameterContextIdIterator.next();
+ final ParameterContextEntity parameterContextEntity = parameterContextMap.get(parameterContextId);
+ try {
+ final String version = String.valueOf(parameterContextEntity.getRevision().getVersion());
+ nifiClient.getParamContextClient().deleteParamContext(parameterContextId, version, true);
+ parameterContextIdIterator.remove();
+ } catch (final Exception e) {
+ logger.warn("Failed to delete parameter context [{}] due to: {}", parameterContextId, e.getMessage());
+ }
+ }
+ }
+ }
+
+ public void deleteParameterProviders() throws NiFiClientException, IOException {
+ final ParameterProvidersEntity parameterProvidersEntity = nifiClient.getFlowClient().getParamProviders();
+ for (final ParameterProviderEntity parameterProviderEntity : parameterProvidersEntity.getParameterProviders()) {
+ final String version = String.valueOf(parameterProviderEntity.getRevision().getVersion());
+ nifiClient.getParamProviderClient().deleteParamProvider(parameterProviderEntity.getId(), version, true);
+ }
+ }
+
public void waitForFlowAnalysisRuleState(final String desiredState, final Collection<String> ruleIdsOfInterest) throws NiFiClientException, IOException {
final long maxTimestamp = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(2L);
diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java
index 57e2dbe0c6..cb6e955d6d 100644
--- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java
+++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java
@@ -249,6 +249,8 @@ public abstract class NiFiSystemIT implements NiFiInstanceProvider {
getClientUtil().deleteControllerLevelServices();
getClientUtil().deleteReportingTasks();
getClientUtil().deleteFlowAnalysisRules();
+ getClientUtil().deleteParameterContexts();
+ getClientUtil().deleteParameterProviders();
logger.info("Finished destroyFlow");
}
diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java
index 265271dc0a..607a264bca 100644
--- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java
+++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java
@@ -32,6 +32,8 @@ public interface ParamContextClient {
ParameterContextEntity deleteParamContext(String id, String version) throws NiFiClientException, IOException;
+ ParameterContextEntity deleteParamContext(String id, String version, boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException;
+
ParameterContextUpdateRequestEntity updateParamContext(ParameterContextEntity paramContext) throws NiFiClientException, IOException;
ParameterContextUpdateRequestEntity getParamContextUpdateRequest(String contextId, String updateRequestId) throws NiFiClientException, IOException;
diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java
index ea17aaa57a..9a7be680de 100644
--- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java
+++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java
@@ -32,6 +32,8 @@ public interface ParamProviderClient {
ParameterProviderEntity deleteParamProvider(String id, String version) throws NiFiClientException, IOException;
+ ParameterProviderEntity deleteParamProvider(String id, String version, boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException;
+
ParameterProviderEntity fetchParameters(ParameterProviderParameterFetchEntity parameterFetchEntity) throws NiFiClientException, IOException;
ParameterProviderApplyParametersRequestEntity applyParameters(ParameterProviderParameterApplicationEntity parameterApplicationEntity) throws NiFiClientException, IOException;
diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java
index 27dd8f53fb..0b25c65407 100644
--- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java
+++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java
@@ -83,6 +83,11 @@ public class JerseyParamContextClient extends AbstractJerseyClient implements Pa
@Override
public ParameterContextEntity deleteParamContext(final String id, final String version) throws NiFiClientException, IOException {
+ return deleteParamContext(id, version, false);
+ }
+
+ @Override
+ public ParameterContextEntity deleteParamContext(final String id, final String version, final boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException {
if (StringUtils.isBlank(id)) {
throw new IllegalArgumentException("Parameter context id cannot be null or blank");
}
@@ -92,9 +97,14 @@ public class JerseyParamContextClient extends AbstractJerseyClient implements Pa
}
return executeAction("Error deleting parameter context", () -> {
- final WebTarget target = paramContextTarget.path("{id}")
+ WebTarget target = paramContextTarget.path("{id}")
.resolveTemplate("id", id)
.queryParam("version", version);
+
+ if (disconnectedNodeAcknowledged) {
+ target = target.queryParam("disconnectedNodeAcknowledged", "true");
+ }
+
return getRequestBuilder(target).delete(ParameterContextEntity.class);
});
}
diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java
index 6026c9c5b3..698e5279e1 100644
--- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java
+++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java
@@ -75,6 +75,11 @@ public class JerseyParamProviderClient extends AbstractJerseyClient implements P
@Override
public ParameterProviderEntity deleteParamProvider(final String id, final String version) throws NiFiClientException, IOException {
+ return deleteParamProvider(id, version, false);
+ }
+
+ @Override
+ public ParameterProviderEntity deleteParamProvider(final String id, final String version, final boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException {
if (StringUtils.isBlank(id)) {
throw new IllegalArgumentException("Parameter provider id cannot be null or blank");
}
@@ -84,9 +89,14 @@ public class JerseyParamProviderClient extends AbstractJerseyClient implements P
}
return executeAction("Error deleting parameter provider", () -> {
- final WebTarget target = paramProviderTarget.path("{id}")
+ WebTarget target = paramProviderTarget.path("{id}")
.resolveTemplate("id", id)
.queryParam("version", version);
+
+ if (disconnectedNodeAcknowledged) {
+ target = target.queryParam("disconnectedNodeAcknowledged", "true");
+ }
+
return getRequestBuilder(target).delete(ParameterProviderEntity.class);
});
}