You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ij...@apache.org on 2018/05/23 00:15:30 UTC
nifi git commit: NIFI-5109 Reset justElectedPrimaryNode flag right
after reelection happen
Repository: nifi
Updated Branches:
refs/heads/master 470d85042 -> dd8b25ab4
NIFI-5109 Reset justElectedPrimaryNode flag right after reelection happen
Read full cluster state and return if no update required
This closes #2657.
Signed-off-by: Koji Kawamura <ij...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/dd8b25ab
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/dd8b25ab
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/dd8b25ab
Branch: refs/heads/master
Commit: dd8b25ab48314dde8cad351a9b564196d807d90e
Parents: 470d850
Author: Max Viazovskyi <il...@gmail.com>
Authored: Mon May 21 21:59:22 2018 +0000
Committer: Koji Kawamura <ij...@apache.org>
Committed: Wed May 23 08:52:49 2018 +0900
----------------------------------------------------------------------
.../util/list/AbstractListProcessor.java | 8 +++--
.../util/list/ITAbstractListProcessor.java | 38 ++++++++++++++++++++
2 files changed, 44 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/dd8b25ab/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/list/AbstractListProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/list/AbstractListProcessor.java b/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/list/AbstractListProcessor.java
index 31ba380..5c42f81 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/list/AbstractListProcessor.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/main/java/org/apache/nifi/processor/util/list/AbstractListProcessor.java
@@ -359,6 +359,7 @@ public abstract class AbstractListProcessor<T extends ListableEntity> extends Ab
if (this.lastListedLatestEntryTimestampMillis == null || this.lastProcessedLatestEntryTimestampMillis == null || justElectedPrimaryNode) {
try {
+ boolean noUpdateRequired = false;
// Attempt to retrieve state from the state manager if a last listing was not yet established or
// if just elected the primary node
final StateMap stateMap = context.getStateManager().getState(getStateScope(context));
@@ -374,8 +375,7 @@ public abstract class AbstractListProcessor<T extends ListableEntity> extends Ab
minTimestampToListMillis = Long.parseLong(v);
// If our determined timestamp is the same as that of our last listing, skip this execution as there are no updates
if (minTimestampToListMillis.equals(this.lastListedLatestEntryTimestampMillis)) {
- context.yield();
- return;
+ noUpdateRequired = true;
} else {
this.lastListedLatestEntryTimestampMillis = minTimestampToListMillis;
}
@@ -386,6 +386,10 @@ public abstract class AbstractListProcessor<T extends ListableEntity> extends Ab
}
}
justElectedPrimaryNode = false;
+ if (noUpdateRequired) {
+ context.yield();
+ return;
+ }
} catch (final IOException ioe) {
getLogger().error("Failed to retrieve timestamp of last listing from the State Manager. Will not perform listing until this is accomplished.");
context.yield();
http://git-wip-us.apache.org/repos/asf/nifi/blob/dd8b25ab/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/test/java/org/apache/nifi/processor/util/list/ITAbstractListProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/test/java/org/apache/nifi/processor/util/list/ITAbstractListProcessor.java b/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/test/java/org/apache/nifi/processor/util/list/ITAbstractListProcessor.java
index dcf47c6..4082ce3 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/test/java/org/apache/nifi/processor/util/list/ITAbstractListProcessor.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-processor-utils/src/test/java/org/apache/nifi/processor/util/list/ITAbstractListProcessor.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.processor.util.list;
+import org.apache.nifi.annotation.notification.PrimaryNodeState;
import org.apache.nifi.components.state.Scope;
import org.apache.nifi.components.state.StateMap;
import org.apache.nifi.flowfile.FlowFile;
@@ -376,6 +377,43 @@ public class ITAbstractListProcessor {
}
@Test
+ public void testResumeListingAfterBecamePrimary() throws Exception {
+ final long initialTimestamp = System.currentTimeMillis();
+
+ runner.assertAllFlowFilesTransferred(ConcreteListProcessor.REL_SUCCESS, 0);
+ proc.addEntity("name", "id", initialTimestamp);
+ proc.addEntity("name", "id2", initialTimestamp);
+
+ // Add entities but these should not be transferred as they are the latest values
+ runner.run();
+ runner.assertAllFlowFilesTransferred(ConcreteListProcessor.REL_SUCCESS, 0);
+ runner.clearTransferState();
+
+ // after providing a pause in listings, the files should now transfer
+ Thread.sleep(DEFAULT_SLEEP_MILLIS);
+
+ runner.run();
+ runner.assertAllFlowFilesTransferred(ConcreteListProcessor.REL_SUCCESS, 2);
+ runner.clearTransferState();
+
+ // Emulate reelection process
+ proc.onPrimaryNodeChange(PrimaryNodeState.ELECTED_PRIMARY_NODE);
+
+ // Now a new file enters
+ proc.addEntity("name", "id3", initialTimestamp + 1);;
+
+ // First run skips the execution because determined timestamp is the same as last listing
+ runner.run();
+ runner.assertAllFlowFilesTransferred(ConcreteListProcessor.REL_SUCCESS, 0);
+ runner.clearTransferState();
+
+ // Now the cluster state has been read, all set to perform next listing
+ runner.run();
+ runner.assertAllFlowFilesTransferred(ConcreteListProcessor.REL_SUCCESS, 1);
+ runner.clearTransferState();
+ }
+
+ @Test
public void testOnlyNewStateStored() throws Exception {
runner.run();