You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2020/10/07 17:57:11 UTC
[nifi] branch main updated: NIFI-7777 Added optional Password
property to UnpackContent for decrypting Zip archives
This is an automated email from the ASF dual-hosted git repository.
alopresto pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new ea6b011 NIFI-7777 Added optional Password property to UnpackContent for decrypting Zip archives
ea6b011 is described below
commit ea6b01159d16a1d14e611f3a211caa42fe5577f2
Author: exceptionfactory <ex...@gmail.com>
AuthorDate: Mon Oct 5 20:28:41 2020 -0400
NIFI-7777 Added optional Password property to UnpackContent for decrypting Zip archives
This closes #4572.
Signed-off-by: Andy LoPresto <al...@apache.org>
---
nifi-assembly/NOTICE | 5 ++
.../nifi-standard-processors/pom.xml | 5 ++
.../nifi/processors/standard/UnpackContent.java | 54 ++++++++++++++++++----
.../processors/standard/TestUnpackContent.java | 54 ++++++++++++++++++++++
4 files changed, 108 insertions(+), 10 deletions(-)
diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE
index b6ee96c..177f0ea 100644
--- a/nifi-assembly/NOTICE
+++ b/nifi-assembly/NOTICE
@@ -1946,6 +1946,11 @@ The following binary components are provided under the Apache Software License v
Apache FtpServer Core 1.1.1
Copyright 2003-2017 The Apache Software Foundation
+ (ASLv2) Zip4j
+ The following NOTICE information applies:
+ Zip4j 2.6.3.
+ Copyright 2020 Srikanth Reddy Lingala
+
************************
Common Development and Distribution License 1.1
************************
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 231e8eb..31d34aa 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -293,6 +293,11 @@
<artifactId>snappy-java</artifactId>
</dependency>
<dependency>
+ <groupId>net.lingala.zip4j</groupId>
+ <artifactId>zip4j</artifactId>
+ <version>2.6.3</version>
+ </dependency>
+ <dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>test</scope>
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
index f802a5d..d6bf5a3 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
@@ -35,10 +35,11 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
+
+import net.lingala.zip4j.model.LocalFileHeader;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
@@ -53,6 +54,8 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.flowfile.attributes.FragmentAttributes;
@@ -72,6 +75,7 @@ import org.apache.nifi.util.FlowFileUnpackager;
import org.apache.nifi.util.FlowFileUnpackagerV1;
import org.apache.nifi.util.FlowFileUnpackagerV2;
import org.apache.nifi.util.FlowFileUnpackagerV3;
+import net.lingala.zip4j.io.inputstream.ZipInputStream;
@EventDriven
@SideEffectFree
@@ -100,7 +104,8 @@ import org.apache.nifi.util.FlowFileUnpackagerV3;
@WritesAttribute(attribute = "file.creationTime", description = "The date and time that the file was created. This attribute holds always the same value as file.lastModifiedTime (tar only)."),
@WritesAttribute(attribute = "file.owner", description = "The owner of the unpacked file (tar only)"),
@WritesAttribute(attribute = "file.group", description = "The group owner of the unpacked file (tar only)"),
- @WritesAttribute(attribute = "file.permissions", description = "The read/write/execute permissions of the unpacked file (tar only)")})
+ @WritesAttribute(attribute = "file.permissions", description = "The read/write/execute permissions of the unpacked file (tar only)"),
+ @WritesAttribute(attribute = "file.encryptionMethod", description = "The encryption method for entries in Zip archives")})
@SeeAlso(MergeContent.class)
public class UnpackContent extends AbstractProcessor {
// attribute keys
@@ -123,6 +128,7 @@ public class UnpackContent extends AbstractProcessor {
public static final String FILE_OWNER_ATTRIBUTE = "file.owner";
public static final String FILE_GROUP_ATTRIBUTE = "file.group";
public static final String FILE_PERMISSIONS_ATTRIBUTE = "file.permissions";
+ public static final String FILE_ENCRYPTION_METHOD_ATTRIBUTE = "file.encryptionMethod";
public static final String FILE_MODIFIED_DATE_ATTR_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern(FILE_MODIFIED_DATE_ATTR_FORMAT).withZone(ZoneId.systemDefault());
@@ -145,6 +151,16 @@ public class UnpackContent extends AbstractProcessor {
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
.build();
+ public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder()
+ .name("Password")
+ .displayName("Password")
+ .description("Password used for decrypting archive entries. Supports Zip files encrypted with ZipCrypto or AES")
+ .required(false)
+ .sensitive(true)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .build();
+
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("Unpacked FlowFiles are sent to this relationship")
@@ -177,6 +193,7 @@ public class UnpackContent extends AbstractProcessor {
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(PACKAGING_FORMAT);
properties.add(FILE_FILTER);
+ properties.add(PASSWORD);
this.properties = Collections.unmodifiableList(properties);
}
@@ -200,7 +217,13 @@ public class UnpackContent extends AbstractProcessor {
if (fileFilter == null) {
fileFilter = Pattern.compile(context.getProperty(FILE_FILTER).getValue());
tarUnpacker = new TarUnpacker(fileFilter);
- zipUnpacker = new ZipUnpacker(fileFilter);
+
+ char[] password = null;
+ final PropertyValue passwordProperty = context.getProperty(PASSWORD);
+ if (passwordProperty.isSet()) {
+ password = passwordProperty.evaluateAttributeExpressions().getValue().toCharArray();
+ }
+ zipUnpacker = new ZipUnpacker(fileFilter, password);
}
}
@@ -301,8 +324,12 @@ public class UnpackContent extends AbstractProcessor {
abstract void unpack(ProcessSession session, FlowFile source, List<FlowFile> unpacked);
- protected boolean fileMatches(ArchiveEntry entry) {
- return fileFilter == null || fileFilter.matcher(entry.getName()).find();
+ protected boolean fileMatches(final ArchiveEntry entry) {
+ return fileMatches(entry.getName());
+ }
+
+ protected boolean fileMatches(final String entryName) {
+ return fileFilter == null || fileFilter.matcher(entryName).find();
}
}
@@ -370,8 +397,11 @@ public class UnpackContent extends AbstractProcessor {
}
private static class ZipUnpacker extends Unpacker {
- public ZipUnpacker(Pattern fileFilter) {
+ private char[] password;
+
+ public ZipUnpacker(final Pattern fileFilter, final char[] password) {
super(fileFilter);
+ this.password = password;
}
@Override
@@ -381,13 +411,14 @@ public class UnpackContent extends AbstractProcessor {
@Override
public void process(final InputStream in) throws IOException {
int fragmentCount = 0;
- try (final ZipArchiveInputStream zipIn = new ZipArchiveInputStream(new BufferedInputStream(in))) {
- ArchiveEntry zipEntry;
+ try (final ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(in), password)) {
+ LocalFileHeader zipEntry;
while ((zipEntry = zipIn.getNextEntry()) != null) {
- if (zipEntry.isDirectory() || !fileMatches(zipEntry)) {
+ final String zipEntryName = zipEntry.getFileName();
+ if (zipEntry.isDirectory() || !fileMatches(zipEntryName)) {
continue;
}
- final File file = new File(zipEntry.getName());
+ final File file = new File(zipEntryName);
final String parentDirectory = (file.getParent() == null) ? "/" : file.getParent();
final Path absPath = file.toPath().toAbsolutePath();
final String absPathString = absPath.getParent().toString() + "/";
@@ -403,6 +434,9 @@ public class UnpackContent extends AbstractProcessor {
attributes.put(FRAGMENT_ID, fragmentId);
attributes.put(FRAGMENT_INDEX, String.valueOf(++fragmentCount));
+ final String encryptionMethod = zipEntry.getEncryptionMethod().toString();
+ attributes.put(FILE_ENCRYPTION_METHOD_ATTRIBUTE, encryptionMethod);
+
unpackedFile = session.putAllAttributes(unpackedFile, attributes);
unpackedFile = session.write(unpackedFile, new OutputStreamCallback() {
@Override
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
index 42cc244..73635bd 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -31,7 +32,11 @@ import java.time.format.DateTimeParseException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
+import net.lingala.zip4j.io.outputstream.ZipOutputStream;
+import net.lingala.zip4j.model.ZipParameters;
+import net.lingala.zip4j.model.enums.EncryptionMethod;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
@@ -222,6 +227,16 @@ public class TestUnpackContent {
}
@Test
+ public void testZipEncryptionZipStandard() throws IOException {
+ runZipEncryptionMethod(EncryptionMethod.ZIP_STANDARD);
+ }
+
+ @Test
+ public void testZipEncryptionAes() throws IOException {
+ runZipEncryptionMethod(EncryptionMethod.AES);
+ }
+
+ @Test
public void testZipWithFilter() throws IOException {
final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent());
final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new UnpackContent());
@@ -450,4 +465,43 @@ public class TestUnpackContent {
runner.assertTransferCount(UnpackContent.REL_SUCCESS, numThreads*2);
}
+
+ private void runZipEncryptionMethod(final EncryptionMethod encryptionMethod) throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new UnpackContent());
+ runner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString());
+
+ final String password = String.class.getSimpleName();
+ runner.setProperty(UnpackContent.PASSWORD, password);
+
+ final char[] streamPassword = password.toCharArray();
+
+ final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ final ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, streamPassword);
+
+ final String name = UUID.randomUUID().toString();
+ final String contents = TestRunner.class.getCanonicalName();
+
+ final ZipParameters zipParameters = new ZipParameters();
+ zipParameters.setEncryptionMethod(encryptionMethod);
+ zipParameters.setEncryptFiles(true);
+ zipParameters.setFileNameInZip(name);
+ zipOutputStream.putNextEntry(zipParameters);
+ zipOutputStream.write(contents.getBytes());
+ zipOutputStream.closeEntry();
+ zipOutputStream.close();
+
+ final byte[] bytes = outputStream.toByteArray();
+ runner.enqueue(bytes);
+ runner.run();
+
+ runner.assertTransferCount(UnpackContent.REL_SUCCESS, 1);
+ runner.assertTransferCount(UnpackContent.REL_ORIGINAL, 1);
+
+ final MockFlowFile unpacked = runner.getFlowFilesForRelationship(UnpackContent.REL_SUCCESS).iterator().next();
+ unpacked.assertAttributeEquals(UnpackContent.FILE_ENCRYPTION_METHOD_ATTRIBUTE, encryptionMethod.toString());
+
+ final byte[] unpackedBytes = runner.getContentAsByteArray(unpacked);
+ final String unpackedContents = new String(unpackedBytes);
+ assertEquals("Unpacked Contents not matched", contents, unpackedContents);
+ }
}