You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by pv...@apache.org on 2018/03/29 08:31:23 UTC

nifi git commit: NIFI-5030: If ControlRate encounters a FlowFile that cannot be transferred, it should continue processing other FlowFiles that have different group attribute values

Repository: nifi
Updated Branches:
  refs/heads/master c1459825b -> bbbe428e7


NIFI-5030: If ControlRate encounters a FlowFile that cannot be transferred, it should continue processing other FlowFiles that have different group attribute values

Signed-off-by: Pierre Villard <pi...@gmail.com>

This closes #2593.


Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/bbbe428e
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/bbbe428e
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/bbbe428e

Branch: refs/heads/master
Commit: bbbe428e7bf6f0a6cc7687cbd750431ae81fad8f
Parents: c145982
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Mar 28 16:06:12 2018 -0400
Committer: Pierre Villard <pi...@gmail.com>
Committed: Thu Mar 29 10:31:15 2018 +0200

----------------------------------------------------------------------
 .../nifi/processors/standard/ControlRate.java   | 17 +++++++--
 .../processors/standard/TestControlRate.java    | 40 ++++++++++++++++++--
 2 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/nifi/blob/bbbe428e/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
index c73f866..cf2364a 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ControlRate.java
@@ -394,18 +394,19 @@ public class ControlRate extends AbstractProcessor {
         @Override
         public FlowFileFilterResult filter(FlowFile flowFile) {
             long accrual = getFlowFileAccrual(flowFile);
-            if(accrual < 0){
+            if (accrual < 0) {
                 // this FlowFile is invalid for this configuration so let the processor deal with it
                 return FlowFileFilterResult.ACCEPT_AND_TERMINATE;
             }
 
-            String groupName = (groupingAttributeName == null) ? DEFAULT_GROUP_ATTRIBUTE : flowFile
-                    .getAttribute(groupingAttributeName);
+            String groupName = (groupingAttributeName == null) ? DEFAULT_GROUP_ATTRIBUTE : flowFile.getAttribute(groupingAttributeName);
+
             // the flow file may not have the required attribute: in this case it is considered part
             // of the DEFAULT_GROUP_ATTRIBUTE
             if (groupName == null) {
                 groupName = DEFAULT_GROUP_ATTRIBUTE;
             }
+
             Throttle throttle = throttleMap.get(groupName);
             if (throttle == null) {
                 throttle = new Throttle(timePeriodSeconds, TimeUnit.SECONDS, getLogger());
@@ -436,7 +437,15 @@ public class ControlRate extends AbstractProcessor {
                 throttle.unlock();
             }
 
-            return FlowFileFilterResult.REJECT_AND_TERMINATE;
+            // If we are not using a grouping attribute, then no FlowFile will be able to continue on. So we can
+            // just TERMINATE the iteration over FlowFiles.
+            // However, if we are using a grouping attribute, then another FlowFile in the queue may be able to proceed,
+            // so we want to continue our iteration.
+            if (groupingAttributeName == null) {
+                return FlowFileFilterResult.REJECT_AND_TERMINATE;
+            }
+
+            return FlowFileFilterResult.REJECT_AND_CONTINUE;
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/nifi/blob/bbbe428e/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestControlRate.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestControlRate.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestControlRate.java
index 0260276..dea87d9 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestControlRate.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestControlRate.java
@@ -16,20 +16,52 @@
  */
 package org.apache.nifi.processors.standard;
 
+import static org.apache.nifi.processors.standard.ControlRate.MAX_FLOW_FILES_PER_BATCH;
+import static org.junit.Assert.assertEquals;
+
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import org.apache.nifi.util.MockFlowFile;
 import org.apache.nifi.util.TestRunner;
 import org.apache.nifi.util.TestRunners;
-
 import org.junit.Test;
 
-import static org.apache.nifi.processors.standard.ControlRate.MAX_FLOW_FILES_PER_BATCH;
-import static org.junit.Assert.assertEquals;
-
 public class TestControlRate {
 
     @Test
+    public void testLimitExceededThenOtherLimitNotExceeded() {
+        // If we have flowfiles queued that have different values for the "Rate Controlled Attribute"
+        // and we encounter a FlowFile whose rate should be throttled, we should continue pulling other flowfiles
+        // whose rate does not need to be throttled.
+        final TestRunner runner = TestRunners.newTestRunner(new ControlRate());
+        runner.setProperty(ControlRate.RATE_CONTROL_CRITERIA, ControlRate.FLOWFILE_RATE);
+        runner.setProperty(ControlRate.MAX_RATE, "3");
+        runner.setProperty(ControlRate.TIME_PERIOD, "1 min");
+        runner.setProperty(ControlRate.GROUPING_ATTRIBUTE_NAME, "group");
+
+        final Map<String, String> group1 = Collections.singletonMap("group", "1");
+        final Map<String, String> group2 = Collections.singletonMap("group", "2");
+
+        for (int i = 0; i < 5; i++) {
+            runner.enqueue("test data", group1);
+        }
+
+        runner.enqueue("test data", group2);
+
+        // Run several times, just to allow the processor to terminate the first poll if it wishes to
+        runner.run();
+
+        runner.assertAllFlowFilesTransferred(ControlRate.REL_SUCCESS, 4);
+
+        final List<MockFlowFile> output = runner.getFlowFilesForRelationship(ControlRate.REL_SUCCESS);
+        assertEquals(3L, output.stream().filter(ff -> ff.getAttribute("group").equals("1")).count());
+        assertEquals(1L, output.stream().filter(ff -> ff.getAttribute("group").equals("2")).count());
+    }
+
+    @Test
     public void testFileCountRate() throws InterruptedException {
         final TestRunner runner = TestRunners.newTestRunner(new ControlRate());
         runner.setProperty(ControlRate.RATE_CONTROL_CRITERIA, ControlRate.FLOWFILE_RATE);