You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2021/02/25 05:57:16 UTC
[nifi] 07/24: NIFI-8200 Modified PutAzureDataLakeStorage to delete
temp file if exception was thrown in uploadContent()
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.13
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit e57bc69e37a8b70d3648b554d5f8576c713d3cc6
Author: Timea Barna <ti...@gmail.com>
AuthorDate: Wed Feb 10 14:27:47 2021 +0100
NIFI-8200 Modified PutAzureDataLakeStorage to delete temp file if exception was thrown in uploadContent()
This closes #4815
Signed-off-by: Joey Frazee <jf...@apache.org>
---
.../azure/storage/PutAzureDataLakeStorage.java | 13 ++++-
.../additionalDetails.html | 57 ++++++++++++++++++++++
.../azure/storage/ITPutAzureDataLakeStorage.java | 17 +++++++
3 files changed, 86 insertions(+), 1 deletion(-)
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureDataLakeStorage.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureDataLakeStorage.java
index 410320d..938252a 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureDataLakeStorage.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureDataLakeStorage.java
@@ -126,6 +126,9 @@ public class PutAzureDataLakeStorage extends AbstractAzureDataLakeStorageProcess
if (length > 0) {
try (final InputStream rawIn = session.read(flowFile); final BufferedInputStream bufferedIn = new BufferedInputStream(rawIn)) {
uploadContent(fileClient, bufferedIn, length);
+ } catch (Exception e) {
+ removeTempFile(fileClient);
+ throw e;
}
}
@@ -162,6 +165,14 @@ public class PutAzureDataLakeStorage extends AbstractAzureDataLakeStorageProcess
}
}
+ private void removeTempFile(DataLakeFileClient fileClient) {
+ try {
+ fileClient.delete();
+ } catch (Exception e) {
+ getLogger().error("Error while removing temp file on Azure Data Lake Storage", e);
+ }
+ }
+
@VisibleForTesting
static void uploadContent(DataLakeFileClient fileClient, InputStream in, long length) {
long chunkStart = 0;
@@ -181,4 +192,4 @@ public class PutAzureDataLakeStorage extends AbstractAzureDataLakeStorageProcess
fileClient.flush(length);
}
-}
\ No newline at end of file
+}
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/docs/org.apache.nifi.processors.azure.storage.PutAzureDataLakeStorage/additionalDetails.html b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/docs/org.apache.nifi.processors.azure.storage.PutAzureDataLakeStorage/additionalDetails.html
new file mode 100644
index 0000000..40e78d1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/docs/org.apache.nifi.processors.azure.storage.PutAzureDataLakeStorage/additionalDetails.html
@@ -0,0 +1,57 @@
+<!DOCTYPE html>
+<html lang="en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <meta charset="utf-8" />
+ <title>PutAzureDataLakeStorage</title>
+ <link rel="stylesheet" href="/nifi-docs/css/component-usage.css" type="text/css" />
+</head>
+
+<body>
+
+<p>
+ This processor is responsible for uploading files to Azure Data Lake Storage Gen2.
+</p>
+
+<h3>File uploading and cleanup process</h3>
+
+<h4>New file</h4>
+
+<ol>
+ <li>An empty file is created.</li>
+ <li>Content is appended to file.</li>
+ <li>In case append failure the file is deleted.</li>
+ <li>In case file deletion failure the empty file remains on the server.</li>
+</ol>
+
+<h4>Existing file</h4>
+
+<ul>
+ <li>Processors with "fail" conflict resolution strategy will be directed to "Failure" relationship.</li>
+ <li>Processors with "ignore" conflict resolution strategy will be directed to "Success" relationship.</li>
+ <li>Processors with "replace" conflict resolution strategy:</li>
+
+ <ol>
+ <li>An empty file overwrites the existing file, the original file is lost.</li>
+ <li>Content is appended to file.</li>
+ <li>In case append failure the file is deleted.</li>
+ <li>In case file deletion failure the empty file remains on the server.</li>
+ </ol>
+</ul>
+
+</body>
+</html>
+
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureDataLakeStorage.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureDataLakeStorage.java
index fa3c684..4854c78 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureDataLakeStorage.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureDataLakeStorage.java
@@ -29,6 +29,7 @@ import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
+import java.io.InputStream;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -43,6 +44,11 @@ import static org.apache.nifi.processors.azure.storage.utils.ADLSAttributes.ATTR
import static org.apache.nifi.processors.azure.storage.utils.ADLSAttributes.ATTR_NAME_PRIMARY_URI;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
public class ITPutAzureDataLakeStorage extends AbstractAzureDataLakeStorageIT {
@@ -241,6 +247,17 @@ public class ITPutAzureDataLakeStorage extends AbstractAzureDataLakeStorageIT {
assertFailure();
}
+ @Test(expected = NullPointerException.class)
+ public void testPutFileButFailedToAppend() {
+ DataLakeFileClient fileClient = mock(DataLakeFileClient.class);
+ InputStream stream = mock(InputStream.class);
+ doThrow(NullPointerException.class).when(fileClient).append(any(InputStream.class), anyLong(), anyLong());
+
+ PutAzureDataLakeStorage.uploadContent(fileClient, stream, FILE_DATA.length);
+
+ verify(fileClient).delete();
+ }
+
private Map<String, String> createAttributesMap() {
Map<String, String> attributes = new HashMap<>();