You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2019/03/12 19:40:09 UTC
[nifi] 17/21: NIFI-6055 Improving error handling during
auto-loading of NARs and ensuring only one Jetty NAR is loaded
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.9.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 4f01a5cc6bfd5dae1a3a309586e92bb7bbe97179
Author: Bryan Bende <bb...@apache.org>
AuthorDate: Wed Feb 20 10:38:46 2019 -0500
NIFI-6055 Improving error handling during auto-loading of NARs and ensuring only one Jetty NAR is loaded
NIFI-6056 Fixing cassandra bundle so nifi-mock is a test scoped and removing incorrect usage of ProviderCreation exception from nifi-framework-api
This closes #3322.
Signed-off-by: Mark Payne <ma...@hotmail.com>
---
.../nifi-cassandra-processors/pom.xml | 6 ++-
.../cassandra/AbstractCassandraProcessor.java | 3 +-
.../cassandra/AbstractCassandraProcessorTest.java | 3 +-
.../org/apache/nifi/nar/StandardNarLoader.java | 7 ++-
.../nar/StandardExtensionDiscoveringManager.java | 58 ++++++++++++---------
.../java/org/apache/nifi/nar/NarUnpackerTest.java | 19 +++++--
.../NarUnpacker/lib/nifi-jetty-bundle.nar | Bin 0 -> 475 bytes
.../main/java/org/apache/nifi/nar/NarUnpacker.java | 37 +++++++++----
nifi-nar-bundles/pom.xml | 12 +++++
9 files changed, 98 insertions(+), 47 deletions(-)
diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/pom.xml b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/pom.xml
index 638fbdc..a07d9c5 100644
--- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/pom.xml
@@ -75,10 +75,14 @@
<artifactId>nifi-record</artifactId>
<scope>compile</scope>
</dependency>
+ <!-- test scoped in nar-bundles dependency management -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock-record-utils</artifactId>
- <version>1.9.1-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessor.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessor.java
index 8444120..2e2b8a8 100644
--- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessor.java
+++ b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessor.java
@@ -31,7 +31,6 @@ import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
-import org.apache.nifi.authentication.exception.ProviderCreationException;
import org.apache.nifi.cassandra.CassandraSessionProviderService;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
@@ -255,7 +254,7 @@ public abstract class AbstractCassandraProcessor extends AbstractProcessor {
try {
clientAuth = SSLContextService.ClientAuth.valueOf(rawClientAuth);
} catch (final IllegalArgumentException iae) {
- throw new ProviderCreationException(String.format("Unrecognized client auth '%s'. Possible values are [%s]",
+ throw new IllegalStateException(String.format("Unrecognized client auth '%s'. Possible values are [%s]",
rawClientAuth, StringUtils.join(SslContextFactory.ClientAuth.values(), ", ")));
}
}
diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessorTest.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessorTest.java
index a2c64c7..4401cd9 100644
--- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessorTest.java
+++ b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessorTest.java
@@ -23,7 +23,6 @@ import com.datastax.driver.core.Metadata;
import com.datastax.driver.core.Row;
import com.google.common.collect.Sets;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
-import org.apache.nifi.authentication.exception.ProviderCreationException;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.processor.ProcessContext;
@@ -233,7 +232,7 @@ public class AbstractCassandraProcessorTest {
assertNotNull(processor.getCluster());
}
- @Test(expected = ProviderCreationException.class)
+ @Test(expected = IllegalStateException.class)
public void testConnectToCassandraWithSSLBadClientAuth() throws Exception {
SSLContextService sslService = mock(SSLContextService.class);
when(sslService.getIdentifier()).thenReturn("ssl-context");
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/StandardNarLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/StandardNarLoader.java
index 2ddca01..20b566b 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/StandardNarLoader.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/StandardNarLoader.java
@@ -141,7 +141,12 @@ public class StandardNarLoader implements NarLoader {
final String version = attributes.getValue(NarManifestEntry.NAR_VERSION.getManifestName());
if (NarClassLoaders.FRAMEWORK_NAR_ID.equals(narId)) {
- LOGGER.error("Found a framework NAR, will not load {}", new Object[]{narFile.getAbsolutePath()});
+ LOGGER.error("Found a framework NAR, will not auto-load {}", new Object[]{narFile.getAbsolutePath()});
+ return null;
+ }
+
+ if (NarClassLoaders.JETTY_NAR_ID.equals(narId)) {
+ LOGGER.error("Found a Jetty NAR, will not auto-load {}", new Object[]{narFile.getAbsolutePath()});
return null;
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
index 5ca7601..8567f32 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java
@@ -150,38 +150,44 @@ public class StandardExtensionDiscoveringManager implements ExtensionDiscovering
final ServiceLoader<?> serviceLoader = ServiceLoader.load(entry.getKey(), bundle.getClassLoader());
for (final Object o : serviceLoader) {
- // create a cache of temp ConfigurableComponent instances, the initialize here has to happen before the checks below
- if ((isControllerService || isProcessor || isReportingTask) && o instanceof ConfigurableComponent) {
- final ConfigurableComponent configurableComponent = (ConfigurableComponent) o;
- initializeTempComponent(configurableComponent);
-
- final String cacheKey = getClassBundleKey(o.getClass().getCanonicalName(), bundle.getBundleDetails().getCoordinate());
- tempComponentLookup.put(cacheKey, (ConfigurableComponent)o);
- }
-
- // only consider extensions discovered directly in this bundle
- boolean registerExtension = bundle.getClassLoader().equals(o.getClass().getClassLoader());
-
- if (registerExtension) {
- final Class extensionType = o.getClass();
- if (isControllerService && !checkControllerServiceEligibility(extensionType)) {
- registerExtension = false;
- logger.error(String.format(
- "Skipping Controller Service %s because it is bundled with its supporting APIs and requires instance class loading.", extensionType.getName()));
+ try {
+ // create a cache of temp ConfigurableComponent instances, the initialize here has to happen before the checks below
+ if ((isControllerService || isProcessor || isReportingTask) && o instanceof ConfigurableComponent) {
+ final ConfigurableComponent configurableComponent = (ConfigurableComponent) o;
+ initializeTempComponent(configurableComponent);
+
+ final String cacheKey = getClassBundleKey(o.getClass().getCanonicalName(), bundle.getBundleDetails().getCoordinate());
+ tempComponentLookup.put(cacheKey, (ConfigurableComponent) o);
}
- final boolean canReferenceControllerService = (isControllerService || isProcessor || isReportingTask) && o instanceof ConfigurableComponent;
- if (canReferenceControllerService && !checkControllerServiceReferenceEligibility((ConfigurableComponent) o, bundle.getClassLoader())) {
- registerExtension = false;
- logger.error(String.format(
- "Skipping component %s because it is bundled with its referenced Controller Service APIs and requires instance class loading.", extensionType.getName()));
- }
+ // only consider extensions discovered directly in this bundle
+ boolean registerExtension = bundle.getClassLoader().equals(o.getClass().getClassLoader());
if (registerExtension) {
- registerServiceClass(o.getClass(), classNameBundleLookup, bundleCoordinateClassesLookup, bundle, entry.getValue());
+ final Class extensionType = o.getClass();
+ if (isControllerService && !checkControllerServiceEligibility(extensionType)) {
+ registerExtension = false;
+ logger.error(String.format(
+ "Skipping Controller Service %s because it is bundled with its supporting APIs and requires instance class loading.", extensionType.getName()));
+ }
+
+ final boolean canReferenceControllerService = (isControllerService || isProcessor || isReportingTask) && o instanceof ConfigurableComponent;
+ if (canReferenceControllerService && !checkControllerServiceReferenceEligibility((ConfigurableComponent) o, bundle.getClassLoader())) {
+ registerExtension = false;
+ logger.error(String.format(
+ "Skipping component %s because it is bundled with its referenced Controller Service APIs and requires instance class loading.", extensionType.getName()));
+ }
+
+ if (registerExtension) {
+ registerServiceClass(o.getClass(), classNameBundleLookup, bundleCoordinateClassesLookup, bundle, entry.getValue());
+ }
+ }
+ } catch (Exception e) {
+ logger.warn("Failed to register extension {} due to: {}" , new Object[]{o.getClass().getCanonicalName(), e.getMessage()});
+ if (logger.isDebugEnabled()) {
+ logger.debug("", e);
}
}
-
}
classLoaderBundleLookup.put(bundle.getClassLoader(), bundle);
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
index ac666ca..cc7fe74 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
@@ -34,7 +34,9 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+import java.util.stream.Stream;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static org.junit.Assert.assertEquals;
@@ -100,6 +102,7 @@ public class NarUnpackerTest {
Set<String> expectedNars = new HashSet<>();
expectedNars.add("dummy-one.nar-unpacked");
expectedNars.add("dummy-two.nar-unpacked");
+ expectedNars.add("nifi-jetty-bundle.nar-unpacked");
assertEquals(expectedNars.size(), extensionFiles.length);
for (File extensionFile : extensionFiles) {
@@ -127,8 +130,12 @@ public class NarUnpackerTest {
final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
File[] extensionFiles = extensionsWorkingDir.listFiles();
- assertEquals(1, extensionFiles.length);
- assertEquals("dummy-one.nar-unpacked", extensionFiles[0].getName());
+ assertEquals(2, extensionFiles.length);
+
+ final Optional<File> foundDummyOne = Stream.of(extensionFiles)
+ .filter(f -> f.getName().equals("dummy-one.nar-unpacked"))
+ .findFirst();
+ assertTrue(foundDummyOne.isPresent());
}
@Test
@@ -151,8 +158,12 @@ public class NarUnpackerTest {
final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
File[] extensionFiles = extensionsWorkingDir.listFiles();
- assertEquals(1, extensionFiles.length);
- assertEquals("dummy-one.nar-unpacked", extensionFiles[0].getName());
+ assertEquals(2, extensionFiles.length);
+
+ final Optional<File> foundDummyOne = Stream.of(extensionFiles)
+ .filter(f -> f.getName().equals("dummy-one.nar-unpacked"))
+ .findFirst();
+ assertTrue(foundDummyOne.isPresent());
}
@Test
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/NarUnpacker/lib/nifi-jetty-bundle.nar b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/NarUnpacker/lib/nifi-jetty-bundle.nar
new file mode 100644
index 0000000..2adca76
Binary files /dev/null and b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/NarUnpacker/lib/nifi-jetty-bundle.nar differ
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java
index edb824f..8bc163c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java
@@ -16,6 +16,14 @@
*/
package org.apache.nifi.nar;
+import org.apache.nifi.bundle.Bundle;
+import org.apache.nifi.bundle.BundleCoordinate;
+import org.apache.nifi.util.FileUtils;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
@@ -41,13 +49,6 @@ import java.util.jar.Attributes;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
-import org.apache.nifi.bundle.Bundle;
-import org.apache.nifi.bundle.BundleCoordinate;
-import org.apache.nifi.util.FileUtils;
-import org.apache.nifi.util.NiFiProperties;
-import org.apache.nifi.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
*
@@ -72,6 +73,7 @@ public final class NarUnpacker {
final Map<File, BundleCoordinate> unpackedNars = new HashMap<>();
try {
+ File unpackedJetty = null;
File unpackedFramework = null;
final Set<File> unpackedExtensions = new HashSet<>();
final List<File> narFiles = new ArrayList<>();
@@ -119,13 +121,19 @@ public final class NarUnpacker {
// unpack the framework nar
unpackedFramework = unpackNar(narFile, frameworkWorkingDir);
+ } else if (NarClassLoaders.JETTY_NAR_ID.equals(narId)) {
+ if (unpackedJetty != null) {
+ throw new IllegalStateException("Multiple Jetty NARs discovered. Only one Jetty NAR is permitted.");
+ }
+
+ // unpack and record the Jetty nar
+ unpackedJetty = unpackNar(narFile, extensionsWorkingDir);
+ unpackedNars.put(unpackedJetty, new BundleCoordinate(groupId, narId, version));
+ unpackedExtensions.add(unpackedJetty);
} else {
+ // unpack and record the extension nar
final File unpackedExtension = unpackNar(narFile, extensionsWorkingDir);
-
- // record the current bundle
unpackedNars.put(unpackedExtension, new BundleCoordinate(groupId, narId, version));
-
- // unpack the extension nar
unpackedExtensions.add(unpackedExtension);
}
}
@@ -138,6 +146,13 @@ public final class NarUnpacker {
throw new IllegalStateException("Framework NAR cannot be read.");
}
+ // ensure we've found the jetty nar
+ if (unpackedJetty == null) {
+ throw new IllegalStateException("No Jetty NAR found.");
+ } else if (!unpackedJetty.canRead()) {
+ throw new IllegalStateException("Jetty NAR cannot be read.");
+ }
+
// Determine if any nars no longer exist and delete their working directories. This happens
// if a new version of a nar is dropped into the lib dir. ensure no old framework are present
final File[] frameworkWorkingDirContents = frameworkWorkingDir.listFiles();
diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml
index dccf1bf..0ea2714 100755
--- a/nifi-nar-bundles/pom.xml
+++ b/nifi-nar-bundles/pom.xml
@@ -257,6 +257,18 @@
<version>1.9.1-SNAPSHOT</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <version>1.10.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock-record-utils</artifactId>
+ <version>1.10.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
<!-- The following dependencies are marked provided because they must be provided by the container. Nars can assume they are there-->
<dependency>
<groupId>org.apache.nifi</groupId>