You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2023/11/01 18:10:06 UTC
(nifi) branch main updated: NIFI-12294 Standardized NAR Entry Loading (#7958)
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 945d8b54bc NIFI-12294 Standardized NAR Entry Loading (#7958)
945d8b54bc is described below
commit 945d8b54bce63d7454238e11e08d073b88748378
Author: exceptionfactory <ex...@apache.org>
AuthorDate: Wed Nov 1 13:10:00 2023 -0500
NIFI-12294 Standardized NAR Entry Loading (#7958)
- Consolidated duplicative NAR file entry normalization
---
.../java/org/apache/nifi/nar/NarUnpackerTest.java | 49 +++++------
.../main/java/org/apache/nifi/nar/NarUnpacker.java | 99 +++++++++++-----------
2 files changed, 72 insertions(+), 76 deletions(-)
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
index beb673f0a8..5733d1bb60 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
@@ -33,6 +33,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
@@ -44,17 +45,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class NarUnpackerTest {
+ private static final String PROPERTIES_PATH = "/NarUnpacker/conf/nifi.properties";
+
@BeforeAll
public static void copyResources() throws IOException {
-
final Path sourcePath = Paths.get("./src/test/resources");
final Path targetPath = Paths.get("./target");
- Files.walkFileTree(sourcePath, new SimpleFileVisitor<Path>() {
+ Files.walkFileTree(sourcePath, new SimpleFileVisitor<>() {
@Override
- public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
- throws IOException {
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
Path relativeSource = sourcePath.relativize(dir);
Path target = targetPath.resolve(relativeSource);
@@ -62,12 +63,10 @@ public class NarUnpackerTest {
Files.createDirectories(target);
return FileVisitResult.CONTINUE;
-
}
@Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
- throws IOException {
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Path relativeSource = sourcePath.relativize(file);
Path target = targetPath.resolve(relativeSource);
@@ -81,8 +80,7 @@ public class NarUnpackerTest {
@Test
public void testUnpackNars() {
-
- NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties", Collections.EMPTY_MAP);
+ NiFiProperties properties = loadSpecifiedProperties(Collections.emptyMap());
assertEquals("./target/NarUnpacker/lib/",
properties.getProperty("nifi.nar.library.directory"));
@@ -93,10 +91,10 @@ public class NarUnpackerTest {
assertEquals(2, extensionMapping.getAllExtensionNames().size());
- assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.one"));
- assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.two"));
+ assertTrue(extensionMapping.getAllExtensionNames().containsKey("org.apache.nifi.processors.dummy.one"));
+ assertTrue(extensionMapping.getAllExtensionNames().containsKey("org.apache.nifi.processors.dummy.two"));
final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
- File[] extensionFiles = extensionsWorkingDir.listFiles();
+ File[] extensionFiles = Objects.requireNonNull(extensionsWorkingDir.listFiles());
Set<String> expectedNars = new HashSet<>();
expectedNars.add("dummy-one.nar-unpacked");
@@ -110,24 +108,22 @@ public class NarUnpackerTest {
}
@Test
- public void testUnpackNarsFromEmptyDir() throws IOException {
-
+ public void testUnpackNarsFromEmptyDir() {
final File emptyDir = new File("./target/empty/dir");
- emptyDir.delete();
emptyDir.deleteOnExit();
assertTrue(emptyDir.mkdirs());
final Map<String, String> others = new HashMap<>();
others.put("nifi.nar.library.directory.alt", emptyDir.toString());
- NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties", others);
+ NiFiProperties properties = loadSpecifiedProperties(others);
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties, SystemBundle.create(properties), NarUnpackMode.UNPACK_INDIVIDUAL_JARS);
assertEquals(1, extensionMapping.getAllExtensionNames().size());
- assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.one"));
+ assertTrue(extensionMapping.getAllExtensionNames().containsKey("org.apache.nifi.processors.dummy.one"));
final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
- File[] extensionFiles = extensionsWorkingDir.listFiles();
+ File[] extensionFiles = Objects.requireNonNull(extensionsWorkingDir.listFiles());
assertEquals(2, extensionFiles.length);
@@ -139,23 +135,21 @@ public class NarUnpackerTest {
@Test
public void testUnpackNarsFromNonExistantDir() {
-
final File nonExistantDir = new File("./target/this/dir/should/not/exist/");
- nonExistantDir.delete();
nonExistantDir.deleteOnExit();
final Map<String, String> others = new HashMap<>();
others.put("nifi.nar.library.directory.alt", nonExistantDir.toString());
- NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties", others);
+ NiFiProperties properties = loadSpecifiedProperties(others);
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties, SystemBundle.create(properties), NarUnpackMode.UNPACK_INDIVIDUAL_JARS);
- assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.one"));
+ assertTrue(extensionMapping.getAllExtensionNames().containsKey("org.apache.nifi.processors.dummy.one"));
assertEquals(1, extensionMapping.getAllExtensionNames().size());
final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
- File[] extensionFiles = extensionsWorkingDir.listFiles();
+ File[] extensionFiles = Objects.requireNonNull(extensionsWorkingDir.listFiles());
assertEquals(2, extensionFiles.length);
@@ -167,24 +161,23 @@ public class NarUnpackerTest {
@Test
public void testUnpackNarsFromNonDir() throws IOException {
-
final File nonDir = new File("./target/file.txt");
- nonDir.createNewFile();
+ assertTrue(nonDir.createNewFile());
nonDir.deleteOnExit();
final Map<String, String> others = new HashMap<>();
others.put("nifi.nar.library.directory.alt", nonDir.toString());
- NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties", others);
+ NiFiProperties properties = loadSpecifiedProperties(others);
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties, SystemBundle.create(properties), NarUnpackMode.UNPACK_INDIVIDUAL_JARS);
assertNull(extensionMapping);
}
- private NiFiProperties loadSpecifiedProperties(final String propertiesFile, final Map<String, String> others) {
+ private NiFiProperties loadSpecifiedProperties(final Map<String, String> others) {
String filePath;
try {
- filePath = NarUnpackerTest.class.getResource(propertiesFile).toURI().getPath();
+ filePath = Objects.requireNonNull(NarUnpackerTest.class.getResource(PROPERTIES_PATH)).toURI().getPath();
} catch (URISyntaxException ex) {
throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex);
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java
index 52b0dd58a0..7f2a49014c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java
@@ -40,6 +40,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
@@ -55,13 +56,11 @@ import java.util.jar.JarInputStream;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
-import static java.lang.String.format;
-
-/**
- *
- */
public final class NarUnpacker {
public static final String BUNDLED_DEPENDENCIES_DIRECTORY = "NAR-INF/bundled-dependencies";
+
+ private static final String BUNDLED_DEPENDENCIES_PREFIX = "META-INF/bundled-dependencies";
+
private static final Logger logger = LoggerFactory.getLogger(NarUnpacker.class);
private static final String HASH_FILENAME = "nar-digest";
private static final FileFilter NAR_FILTER = pathname -> {
@@ -93,8 +92,6 @@ public final class NarUnpacker {
final boolean requireFrameworkNar, final String frameworkNarId,
final boolean requireJettyNar, final boolean verifyHash, final NarUnpackMode unpackMode,
final Predicate<BundleCoordinate> narFilter) {
- final Map<File, BundleCoordinate> unpackedNars = new HashMap<>();
-
try {
File unpackedJetty = null;
File unpackedFramework = null;
@@ -128,13 +125,13 @@ public final class NarUnpacker {
if (!narFiles.isEmpty()) {
final long startTime = System.nanoTime();
- logger.info("Expanding " + narFiles.size() + " NAR files with all processors...");
+ logger.info("Expanding {} NAR files started", narFiles.size());
for (File narFile : narFiles) {
if (!narFile.canRead()) {
throw new IllegalStateException("Unable to read NAR file: " + narFile.getAbsolutePath());
}
- logger.debug("Expanding NAR file: " + narFile.getAbsolutePath());
+ logger.debug("Expanding NAR file: {}", narFile.getAbsolutePath());
// get the manifest for this nar
try (final JarFile nar = new JarFile(narFile)) {
@@ -211,11 +208,11 @@ public final class NarUnpacker {
}
final long duration = System.nanoTime() - startTime;
- logger.info("NAR loading process took " + duration + " nanoseconds "
- + "(" + (int) TimeUnit.SECONDS.convert(duration, TimeUnit.NANOSECONDS) + " seconds).");
+ final double durationSeconds = TimeUnit.NANOSECONDS.toMillis(duration) / 1000.0;
+ logger.info("Expanded {} NAR files in {} seconds ({} ns)", narFiles.size(), durationSeconds, duration);
}
- unpackedNars.putAll(createUnpackedNarBundleCoordinateMap(extensionsWorkingDir));
+ final Map<File, BundleCoordinate> unpackedNars = new HashMap<>(createUnpackedNarBundleCoordinateMap(extensionsWorkingDir));
final ExtensionMapping extensionMapping = new ExtensionMapping();
mapExtensions(unpackedNars, docsWorkingDir, extensionMapping);
@@ -224,7 +221,7 @@ public final class NarUnpacker {
return extensionMapping;
} catch (IOException e) {
- logger.warn("Unable to load NAR library bundles due to {} Will proceed without loading any further Nar bundles", e.toString(), e);
+ logger.warn("Unable to load NAR bundles. Proceeding without loading any further NAR bundles", e);
}
return null;
@@ -236,8 +233,12 @@ public final class NarUnpacker {
* @return map of coordinates for bundles
*/
private static Map<File, BundleCoordinate> createUnpackedNarBundleCoordinateMap(File extensionsWorkingDir) {
- Map<File, BundleCoordinate> result = new HashMap<>();
File[] unpackedDirs = extensionsWorkingDir.listFiles(file -> file.isDirectory() && file.getName().endsWith("nar-unpacked"));
+ if (unpackedDirs == null) {
+ return Collections.emptyMap();
+ }
+
+ final Map<File, BundleCoordinate> result = new HashMap<>();
for (File unpackedDir : unpackedDirs) {
Path mf = Paths.get(unpackedDir.getAbsolutePath(), "META-INF", "MANIFEST.MF");
try(InputStream is = Files.newInputStream(mf)) {
@@ -245,19 +246,18 @@ public final class NarUnpacker {
BundleCoordinate bundleCoordinate = createBundleCoordinate(manifest);
result.put(unpackedDir, bundleCoordinate);
} catch (IOException e) {
- logger.error(format("Unable to parse NAR information from unpacked nar directory [%s].", unpackedDir.getAbsoluteFile()), e);
+ logger.error("Unable to parse NAR information from unpacked directory [{}]", unpackedDir.getAbsoluteFile(), e);
}
}
return result;
}
- private static BundleCoordinate createBundleCoordinate(Manifest manifest) {
+ private static BundleCoordinate createBundleCoordinate(final Manifest manifest) {
Attributes mainAttributes = manifest.getMainAttributes();
String groupId = mainAttributes.getValue(NarManifestEntry.NAR_GROUP.getManifestName());
String narId = mainAttributes.getValue(NarManifestEntry.NAR_ID.getManifestName());
String version = mainAttributes.getValue(NarManifestEntry.NAR_VERSION.getManifestName());
- BundleCoordinate bundleCoordinate = new BundleCoordinate(groupId, narId, version);
- return bundleCoordinate;
+ return new BundleCoordinate(groupId, narId, version);
}
private static void mapExtensions(final Map<File, BundleCoordinate> unpackedNars, final File docsDirectory, final ExtensionMapping mapping) throws IOException {
@@ -319,7 +319,7 @@ public final class NarUnpacker {
} else {
final byte[] hashFileContents = Files.readAllBytes(workingHashFile.toPath());
if (!Arrays.equals(hashFileContents, narDigest)) {
- logger.info("Contents of nar {} have changed. Reloading.", new Object[] { nar.getAbsolutePath() });
+ logger.info("Reloading changed NAR [{}]", nar.getAbsolutePath());
FileUtils.deleteFile(narWorkingDirectory, true);
unpackIndividualJars(nar, narWorkingDirectory, narDigest, unpackMode);
}
@@ -353,16 +353,11 @@ public final class NarUnpacker {
final Enumeration<JarEntry> jarEntries = jarFile.entries();
while (jarEntries.hasMoreElements()) {
final JarEntry jarEntry = jarEntries.nextElement();
- String name = jarEntry.getName();
- if (name.contains("META-INF/bundled-dependencies")){
- name = name.replace("META-INF/bundled-dependencies", BUNDLED_DEPENDENCIES_DIRECTORY);
- }
-
- final File f = new File(workingDirectory, name);
+ final File jarEntryFile = getMappedJarEntryFile(workingDirectory, jarEntry);
if (jarEntry.isDirectory()) {
- FileUtils.ensureDirectoryExistAndCanReadAndWrite(f);
+ FileUtils.ensureDirectoryExistAndCanReadAndWrite(jarEntryFile);
} else {
- makeFile(jarFile.getInputStream(jarEntry), f);
+ makeFile(jarFile.getInputStream(jarEntry), jarEntryFile);
}
}
}
@@ -398,31 +393,27 @@ public final class NarUnpacker {
final Enumeration<JarEntry> jarEntries = jarFile.entries();
while (jarEntries.hasMoreElements()) {
final JarEntry jarEntry = jarEntries.nextElement();
- String name = jarEntry.getName();
- if (name.contains("META-INF/bundled-dependencies")){
- name = name.replace("META-INF/bundled-dependencies", BUNDLED_DEPENDENCIES_DIRECTORY);
- }
-
- logger.debug("Unpacking NAR entry {}", name);
+ final File jarEntryFile = getMappedJarEntryFile(workingDirectory, jarEntry);
+ logger.debug("Unpacking NAR entry {}", jarEntryFile);
// If we've not yet created this entry, create it now. If we've already created the entry, ignore it.
- if (!entriesCreated.add(name)) {
+ if (!entriesCreated.add(jarEntry.getName())) {
continue;
}
// Explode anything from META-INF and any WAR files into the nar's output directory instead of copying it to the uber jar.
// The WAR files are important so that NiFi can load its UI. The META-INF/ directory is important in order to ensure that our
// NarClassLoader has all of the information that it needs.
- if (name.contains("META-INF/") || (name.contains("NAR-INF") && name.endsWith(".war"))) {
+ final String jarEntryFilePath = jarEntryFile.getAbsolutePath();
+ if (jarEntryFilePath.contains("META-INF") || (jarEntryFilePath.contains("NAR-INF") && jarEntryFilePath.endsWith(".war"))) {
if (jarEntry.isDirectory()) {
continue;
}
- final File outFile = new File(workingDirectory, name);
- Files.createDirectories(outFile.getParentFile().toPath());
+ Files.createDirectories(jarEntryFile.getParentFile().toPath());
try (final InputStream entryIn = jarFile.getInputStream(jarEntry);
- final OutputStream manifestOut = new FileOutputStream(outFile)) {
+ final OutputStream manifestOut = new FileOutputStream(jarEntryFile)) {
copy(entryIn, manifestOut);
}
@@ -431,9 +422,9 @@ public final class NarUnpacker {
if (jarEntry.isDirectory()) {
uberJarOut.putNextEntry(new JarEntry(jarEntry.getName()));
- } else if (name.endsWith(".jar")) {
+ } else if (jarEntryFilePath.endsWith(".jar")) {
// Unpack each .jar file into the uber jar, taking care to deal with META-INF/ files, etc. carefully.
- logger.debug("Unpacking Jar {}", name);
+ logger.debug("Unpacking JAR {}", jarEntryFile);
try (final InputStream entryIn = jarFile.getInputStream(jarEntry);
final InputStream in = new BufferedInputStream(entryIn)) {
@@ -475,6 +466,7 @@ public final class NarUnpacker {
JarEntry jarEntry;
while ((jarEntry = jarInputStream.getNextJarEntry()) != null) {
final String entryName = jarEntry.getName();
+ final File outFile = getJarEntryFile(workingDirectory, entryName);
// The META-INF/ directory can contain several different types of files. For example, it contains:
// MANIFEST.MF
@@ -491,8 +483,6 @@ public final class NarUnpacker {
if ((entryName.contains("META-INF/") && !entryName.contains("META-INF/MANIFEST.MF") ) && !jarEntry.isDirectory()) {
logger.debug("Found META-INF/services file {}", entryName);
- final File outFile = new File(workingDirectory, entryName);
-
// Because we're combining multiple jar files into one, we can run into situations where there may be conflicting filenames
// such as 1 jar has a file named META-INF/license and another jar file has a META-INF/license/my-license.txt. We can generally
// just ignore these, though, as they are not necessary in this temporarily created jar file. So we log it at a debug level and
@@ -569,10 +559,12 @@ public final class NarUnpacker {
// go through each entry in this jar
for (final Enumeration<JarEntry> jarEnumeration = jarFile.entries(); jarEnumeration.hasMoreElements();) {
final JarEntry jarEntry = jarEnumeration.nextElement();
+ final File jarEntryFile = getJarEntryFile(docsDirectory, jarEntry.getName());
+ final String jarEntryName = jarEntryFile.getName();
// if this entry is documentation for this component
- if (jarEntry.getName().startsWith(entryName)) {
- final String name = StringUtils.substringAfter(jarEntry.getName(), "docs/");
+ if (jarEntryName.startsWith(entryName)) {
+ final String name = StringUtils.substringAfter(jarEntryName, "docs/");
final String path = coordinate.getGroup() + "/" + coordinate.getId() + "/" + coordinate.getVersion() + "/" + name;
// if this is a directory create it
@@ -581,7 +573,7 @@ public final class NarUnpacker {
// ensure the documentation directory can be created
if (!componentDocsDirectory.exists() && !componentDocsDirectory.mkdirs()) {
- logger.warn("Unable to create docs directory " + componentDocsDirectory.getAbsolutePath());
+ logger.warn("Unable to create docs directory {}", componentDocsDirectory.getAbsolutePath());
break;
}
} else {
@@ -596,9 +588,6 @@ public final class NarUnpacker {
}
}
- /*
- * Returns true if this jar file contains a NiFi component
- */
private static ExtensionMapping determineDocumentedNiFiComponents(final BundleCoordinate coordinate, final File jar) throws IOException {
final ExtensionMapping mapping = new ExtensionMapping();
@@ -670,6 +659,20 @@ public final class NarUnpacker {
}
}
+ private static File getMappedJarEntryFile(final File workingDirectory, final JarEntry jarEntry) {
+ final String jarEntryName = jarEntry.getName().replace(BUNDLED_DEPENDENCIES_PREFIX, BUNDLED_DEPENDENCIES_DIRECTORY);
+ return getJarEntryFile(workingDirectory, jarEntryName);
+ }
+
+ private static File getJarEntryFile(final File workingDirectory, final String jarEntryName) {
+ final Path workingDirectoryPath = workingDirectory.toPath().normalize();
+ final Path jarEntryPath = workingDirectoryPath.resolve(jarEntryName).normalize();
+ if (jarEntryPath.startsWith(workingDirectoryPath)) {
+ return jarEntryPath.toFile();
+ }
+ throw new IllegalArgumentException(String.format("NAR Entry path not valid [%s]", jarEntryName));
+ }
+
private NarUnpacker() {
}
}