You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2020/02/21 22:39:08 UTC

[nifi] branch master updated (eef0470 -> 6ea78c5)

This is an automated email from the ASF dual-hosted git repository.

joewitt pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git.


    from eef0470  NIFI-7175 Fixed core attributes formatting in Developer Guide. (#4066)
     new 1b0e386  NIFI-7114: Fix file leaks in StandardCommsSession and S2S Reporting components
     new 6ea78c5  NIFI-7114: This closes #4069. Update time-sensitive tests in TestLuceneEventIndex

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../provenance/index/lucene/TestLuceneEventIndex.java    |  4 ++--
 .../nifi/reporting/SiteToSiteBulletinReportingTask.java  | 14 +++++++++++---
 .../nifi/reporting/SiteToSiteMetricsReportingTask.java   | 12 ++++++++++--
 .../reporting/SiteToSiteProvenanceReportingTask.java     | 14 +++++++++++---
 .../nifi/reporting/SiteToSiteStatusReportingTask.java    | 14 +++++++++++---
 .../reporting/sink/SiteToSiteReportingRecordSink.java    | 16 ++++++++++------
 .../distributed/cache/client/StandardCommsSession.java   |  2 ++
 7 files changed, 57 insertions(+), 19 deletions(-)


[nifi] 01/02: NIFI-7114: Fix file leaks in StandardCommsSession and S2S Reporting components

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

joewitt pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git

commit 1b0e3865e9ba160bf3b68e33f1938bc6c03229bd
Author: Matthew Burgess <ma...@apache.org>
AuthorDate: Fri Feb 21 16:20:38 2020 -0500

    NIFI-7114: Fix file leaks in StandardCommsSession and S2S Reporting components
    
    Signed-off-by: Joe Witt <jo...@apache.org>
---
 .../nifi/reporting/SiteToSiteBulletinReportingTask.java  | 14 +++++++++++---
 .../nifi/reporting/SiteToSiteMetricsReportingTask.java   | 12 ++++++++++--
 .../reporting/SiteToSiteProvenanceReportingTask.java     | 14 +++++++++++---
 .../nifi/reporting/SiteToSiteStatusReportingTask.java    | 14 +++++++++++---
 .../reporting/sink/SiteToSiteReportingRecordSink.java    | 16 ++++++++++------
 .../distributed/cache/client/StandardCommsSession.java   |  2 ++
 6 files changed, 55 insertions(+), 17 deletions(-)

diff --git a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteBulletinReportingTask.java b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteBulletinReportingTask.java
index 3e07759..1e68687 100644
--- a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteBulletinReportingTask.java
+++ b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteBulletinReportingTask.java
@@ -137,11 +137,12 @@ public class SiteToSiteBulletinReportingTask extends AbstractSiteToSiteReporting
         final JsonArray jsonArray = arrayBuilder.build();
 
         // Send the JSON document for the current batch
+        Transaction transaction = null;
         try {
             // Lazily create SiteToSiteClient to provide a StateManager
             setup(context);
 
-            final Transaction transaction = getClient().createTransaction(TransferDirection.SEND);
+            transaction = getClient().createTransaction(TransferDirection.SEND);
             if (transaction == null) {
                 getLogger().info("All destination nodes are penalized; will attempt to send data later");
                 return;
@@ -162,8 +163,15 @@ public class SiteToSiteBulletinReportingTask extends AbstractSiteToSiteReporting
             final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
             getLogger().info("Successfully sent {} Bulletins to destination in {} ms; Transaction ID = {}; First Event ID = {}",
                     new Object[]{bulletins.size(), transferMillis, transactionId, bulletins.get(0).getId()});
-        } catch (final IOException e) {
-            throw new ProcessException("Failed to send Bulletins to destination due to IOException:" + e.getMessage(), e);
+        } catch (final Exception e) {
+            if (transaction != null) {
+                transaction.error();
+            }
+            if (e instanceof ProcessException) {
+                throw (ProcessException) e;
+            } else {
+                throw new ProcessException("Failed to send Bulletins to destination due to IOException:" + e.getMessage(), e);
+            }
         }
 
         lastSentBulletinId = currMaxId;
diff --git a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteMetricsReportingTask.java b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteMetricsReportingTask.java
index e781dfa..898bbbd 100644
--- a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteMetricsReportingTask.java
+++ b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteMetricsReportingTask.java
@@ -191,12 +191,13 @@ public class SiteToSiteMetricsReportingTask extends AbstractSiteToSiteReportingT
                 data = getData(context, new ByteArrayInputStream(metricsObject.toString().getBytes(StandardCharsets.UTF_8)), attributes);
             }
 
+            Transaction transaction = null;
             try {
                 // Lazily create SiteToSiteClient to provide a StateManager
                 setup(context);
 
                 long start = System.nanoTime();
-                final Transaction transaction = getClient().createTransaction(TransferDirection.SEND);
+                transaction = getClient().createTransaction(TransferDirection.SEND);
                 if (transaction == null) {
                     getLogger().debug("All destination nodes are penalized; will attempt to send data later");
                     return;
@@ -215,7 +216,14 @@ public class SiteToSiteMetricsReportingTask extends AbstractSiteToSiteReportingT
                 final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
                 getLogger().info("Successfully sent metrics to destination in {}ms; Transaction ID = {}", new Object[]{transferMillis, transactionId});
             } catch (final Exception e) {
-                throw new ProcessException("Failed to send metrics to destination due to:" + e.getMessage(), e);
+                if (transaction != null) {
+                    transaction.error();
+                }
+                if (e instanceof ProcessException) {
+                    throw (ProcessException) e;
+                } else {
+                    throw new ProcessException("Failed to send metrics to destination due to:" + e.getMessage(), e);
+                }
             }
 
         } else {
diff --git a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteProvenanceReportingTask.java b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteProvenanceReportingTask.java
index 0ff507c..d382ca3 100644
--- a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteProvenanceReportingTask.java
+++ b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteProvenanceReportingTask.java
@@ -304,11 +304,12 @@ public class SiteToSiteProvenanceReportingTask extends AbstractSiteToSiteReporti
             final JsonArray jsonArray = arrayBuilder.build();
 
             // Send the JSON document for the current batch
+            Transaction transaction = null;
             try {
                 // Lazily create SiteToSiteClient to provide a StateManager
                 setup(context);
 
-                final Transaction transaction = getClient().createTransaction(TransferDirection.SEND);
+                transaction = getClient().createTransaction(TransferDirection.SEND);
                 if (transaction == null) {
                     // Throw an exception to avoid provenance event id will not proceed so that those can be consumed again.
                     throw new ProcessException("All destination nodes are penalized; will attempt to send data later");
@@ -329,8 +330,15 @@ public class SiteToSiteProvenanceReportingTask extends AbstractSiteToSiteReporti
                 final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
                 getLogger().info("Successfully sent {} Provenance Events to destination in {} ms; Transaction ID = {}; First Event ID = {}",
                         new Object[] {events.size(), transferMillis, transactionId, events.get(0).getEventId()});
-            } catch (final IOException e) {
-                throw new ProcessException("Failed to send Provenance Events to destination due to IOException:" + e.getMessage(), e);
+            } catch (final Exception e) {
+                if (transaction != null) {
+                    transaction.error();
+                }
+                if (e instanceof ProcessException) {
+                    throw (ProcessException) e;
+                } else {
+                    throw new ProcessException("Failed to send Provenance Events to destination due to IOException:" + e.getMessage(), e);
+                }
             }
         });
 
diff --git a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteStatusReportingTask.java b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteStatusReportingTask.java
index 31009f8..7464b43 100644
--- a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteStatusReportingTask.java
+++ b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/SiteToSiteStatusReportingTask.java
@@ -161,12 +161,13 @@ public class SiteToSiteStatusReportingTask extends AbstractSiteToSiteReportingTa
 
         while(!jsonBatch.isEmpty()) {
             // Send the JSON document for the current batch
+            Transaction transaction = null;
             try {
                 // Lazily create SiteToSiteClient to provide a StateManager
                 setup(context);
 
                 long start = System.nanoTime();
-                final Transaction transaction = getClient().createTransaction(TransferDirection.SEND);
+                transaction = getClient().createTransaction(TransferDirection.SEND);
                 if (transaction == null) {
                     getLogger().debug("All destination nodes are penalized; will attempt to send data later");
                     return;
@@ -197,8 +198,15 @@ public class SiteToSiteStatusReportingTask extends AbstractSiteToSiteReportingTa
                 fromIndex = toIndex;
                 toIndex = Math.min(fromIndex + batchSize, jsonArray.size());
                 jsonBatch = jsonArray.subList(fromIndex, toIndex);
-            } catch (final IOException e) {
-                throw new ProcessException("Failed to send Status Records to destination due to IOException:" + e.getMessage(), e);
+            } catch (final Exception e) {
+                if (transaction != null) {
+                    transaction.error();
+                }
+                if (e instanceof ProcessException) {
+                    throw (ProcessException) e;
+                } else {
+                    throw new ProcessException("Failed to send Status Records to destination due to IOException:" + e.getMessage(), e);
+                }
             }
         }
     }
diff --git a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/sink/SiteToSiteReportingRecordSink.java b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/sink/SiteToSiteReportingRecordSink.java
index 20c501e..b2ed107 100644
--- a/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/sink/SiteToSiteReportingRecordSink.java
+++ b/nifi-nar-bundles/nifi-site-to-site-reporting-bundle/nifi-site-to-site-reporting-task/src/main/java/org/apache/nifi/reporting/sink/SiteToSiteReportingRecordSink.java
@@ -135,10 +135,10 @@ public class SiteToSiteReportingRecordSink extends AbstractControllerService imp
 
     @Override
     public WriteResult sendData(final RecordSet recordSet, final Map<String,String> attributes, final boolean sendZeroResults) throws IOException {
-
+        Transaction transaction = null;
         try {
             WriteResult writeResult = null;
-            final Transaction transaction = getClient().createTransaction(TransferDirection.SEND);
+            transaction = getClient().createTransaction(TransferDirection.SEND);
             if (transaction == null) {
                 getLogger().info("All destination nodes are penalized; will attempt to send data later");
             } else {
@@ -166,12 +166,16 @@ public class SiteToSiteReportingRecordSink extends AbstractControllerService imp
                 }
             }
             return writeResult;
-        } catch(IOException ioe) {
-            throw ioe;
         } catch (Exception e) {
-            throw new IOException("Failed to write metrics using record writer: " + e.getMessage(), e);
+            if (transaction != null) {
+                transaction.error();
+            }
+            if (e instanceof IOException) {
+                throw (IOException) e;
+            } else {
+                throw new IOException("Failed to write metrics using record writer: " + e.getMessage(), e);
+            }
         }
-
     }
 
     @OnDisabled
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/StandardCommsSession.java b/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/StandardCommsSession.java
index d157161..56e3389 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/StandardCommsSession.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-distributed-cache-services-bundle/nifi-distributed-cache-client-service/src/main/java/org/apache/nifi/distributed/cache/client/StandardCommsSession.java
@@ -71,6 +71,8 @@ public class StandardCommsSession implements CommsSession {
     @Override
     public void close() throws IOException {
         socketChannel.close();
+        bufferedIn.close();
+        bufferedOut.close();
     }
 
     @Override


[nifi] 02/02: NIFI-7114: This closes #4069. Update time-sensitive tests in TestLuceneEventIndex

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

joewitt pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git

commit 6ea78c55480f306ce3f49bc6636e73d11e7e44c5
Author: Matthew Burgess <ma...@apache.org>
AuthorDate: Fri Feb 21 16:54:47 2020 -0500

    NIFI-7114: This closes #4069. Update time-sensitive tests in TestLuceneEventIndex
    
    Signed-off-by: Joe Witt <jo...@apache.org>
---
 .../org/apache/nifi/provenance/index/lucene/TestLuceneEventIndex.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/index/lucene/TestLuceneEventIndex.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/index/lucene/TestLuceneEventIndex.java
index d7cf4f6..bd4cc3d 100644
--- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/index/lucene/TestLuceneEventIndex.java
+++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/index/lucene/TestLuceneEventIndex.java
@@ -459,7 +459,7 @@ public class TestLuceneEventIndex {
 
             final QueryResult result = submission.getResult();
             assertNotNull(result);
-            result.awaitCompletion(100, TimeUnit.MILLISECONDS);
+            result.awaitCompletion(2000, TimeUnit.MILLISECONDS);
 
             assertTrue(result.isFinished());
             assertNull(result.getError());
@@ -502,7 +502,7 @@ public class TestLuceneEventIndex {
 
             final QueryResult result = submission.getResult();
             assertNotNull(result);
-            result.awaitCompletion(100, TimeUnit.MILLISECONDS);
+            result.awaitCompletion(2000, TimeUnit.MILLISECONDS);
 
             assertTrue(result.isFinished());
             assertNull(result.getError());