You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2016/08/14 18:35:42 UTC
[1/2] nifi git commit: NIFI-2560 added connection timeout to IMAP/POP3
Repository: nifi
Updated Branches:
refs/heads/master d3b96dcac -> 9c7668948
NIFI-2560 added connection timeout to IMAP/POP3
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b7b56b1c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b7b56b1c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b7b56b1c
Branch: refs/heads/master
Commit: b7b56b1cade12489309ea6e8843336f82cde4b23
Parents: d3b96dc
Author: Oleg Zhurakousky <ol...@suitcase.io>
Authored: Fri Aug 12 16:20:03 2016 -0400
Committer: joewitt <jo...@apache.org>
Committed: Sun Aug 14 14:06:07 2016 -0400
----------------------------------------------------------------------
.../email/AbstractEmailProcessor.java | 22 ++++++++++++++++++--
1 file changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b7b56b1c/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
index e7fd27c..292d42b 100644
--- a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
+++ b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
@@ -46,6 +46,7 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.util.FormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.support.StaticListableBeanFactory;
@@ -122,6 +123,16 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
.defaultValue("false")
.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
.build();
+ static final PropertyDescriptor CONNECTION_TIMEOUT = new PropertyDescriptor.Builder()
+ .name("connection.timeout")
+ .displayName("Connection timeout")
+ .description("The amount of time to wait to connect to Email server")
+ .required(true)
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .defaultValue("30 sec")
+ .build();
+
static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
@@ -144,6 +155,7 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
SHARED_DESCRIPTORS.add(FOLDER);
SHARED_DESCRIPTORS.add(FETCH_SIZE);
SHARED_DESCRIPTORS.add(SHOULD_DELETE_MESSAGES);
+ SHARED_DESCRIPTORS.add(CONNECTION_TIMEOUT);
SHARED_RELATIONSHIPS.add(REL_SUCCESS);
}
@@ -287,17 +299,23 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
/**
* Extracts dynamic properties which typically represent the Java Mail
- * properties from the {@link ProcessContext} returnining them as instance
+ * properties from the {@link ProcessContext} returning them as instance
* of {@link Properties}
*/
private Properties buildJavaMailProperties(ProcessContext context) {
Properties javaMailProperties = new Properties();
for (Entry<PropertyDescriptor, String> propertyDescriptorEntry : context.getProperties().entrySet()) {
- if (propertyDescriptorEntry.getKey().isDynamic()) {
+ if (propertyDescriptorEntry.getKey().isDynamic()
+ && !propertyDescriptorEntry.getKey().getName().equals("mail.imap.timeout")
+ && !propertyDescriptorEntry.getKey().getName().equals("mail.pop3.timeout")) {
javaMailProperties.setProperty(propertyDescriptorEntry.getKey().getName(),
propertyDescriptorEntry.getValue());
}
}
+ String propertyName = this.getProtocol(context).equals("pop3") ? "mail.pop3.timeout" : "mail.imap.timeout";
+
+ javaMailProperties.setProperty(propertyName, String.valueOf(FormatUtils
+ .getTimeDuration(context.getProperty(CONNECTION_TIMEOUT).getValue().trim(), TimeUnit.MILLISECONDS)));
return javaMailProperties;
}
[2/2] nifi git commit: NIFI-2560 This closes #851. fixed property
handling and added tests using it
Posted by jo...@apache.org.
NIFI-2560 This closes #851. fixed property handling and added tests using it
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9c766894
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9c766894
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9c766894
Branch: refs/heads/master
Commit: 9c7668948c9a1541b9b317ab5352570910037d20
Parents: b7b56b1
Author: joewitt <jo...@apache.org>
Authored: Sun Aug 14 14:35:13 2016 -0400
Committer: joewitt <jo...@apache.org>
Committed: Sun Aug 14 14:35:25 2016 -0400
----------------------------------------------------------------------
.../email/AbstractEmailProcessor.java | 23 ++++++++------------
.../nifi/processors/email/ConsumeEmailTest.java | 23 ++++++++++++++++++++
2 files changed, 32 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/9c766894/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
index 292d42b..62f4a88 100644
--- a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
+++ b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
@@ -46,7 +46,6 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.util.FormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.support.StaticListableBeanFactory;
@@ -58,8 +57,7 @@ import org.springframework.util.StreamUtils;
* Base processor for implementing processors to consume messages from Email
* servers using Spring Integration libraries.
*
- * @param <T>
- * the type of {@link AbstractMailReceiver}.
+ * @param <T> the type of {@link AbstractMailReceiver}.
*/
abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends AbstractProcessor {
@@ -133,15 +131,14 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
.defaultValue("30 sec")
.build();
-
static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("All messages that are the are successfully received from Email server and converted to FlowFiles are routed to this relationship")
.build();
- static List<PropertyDescriptor> SHARED_DESCRIPTORS = new ArrayList<>();
+ final static List<PropertyDescriptor> SHARED_DESCRIPTORS = new ArrayList<>();
- static Set<Relationship> SHARED_RELATIONSHIPS = new HashSet<>();
+ final static Set<Relationship> SHARED_RELATIONSHIPS = new HashSet<>();
/*
* Will ensure that list of PropertyDescriptors is build only once, since
@@ -219,8 +216,7 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
* Delegates to sub-classes to build the target receiver as
* {@link AbstractMailReceiver}
*
- * @param context
- * instance of {@link ProcessContext}
+ * @param context instance of {@link ProcessContext}
* @return new instance of {@link AbstractMailReceiver}
*/
protected abstract T buildMessageReceiver(ProcessContext context);
@@ -299,8 +295,8 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
/**
* Extracts dynamic properties which typically represent the Java Mail
- * properties from the {@link ProcessContext} returning them as instance
- * of {@link Properties}
+ * properties from the {@link ProcessContext} returning them as instance of
+ * {@link Properties}
*/
private Properties buildJavaMailProperties(ProcessContext context) {
Properties javaMailProperties = new Properties();
@@ -313,9 +309,8 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
}
}
String propertyName = this.getProtocol(context).equals("pop3") ? "mail.pop3.timeout" : "mail.imap.timeout";
-
- javaMailProperties.setProperty(propertyName, String.valueOf(FormatUtils
- .getTimeDuration(context.getProperty(CONNECTION_TIMEOUT).getValue().trim(), TimeUnit.MILLISECONDS)));
+ final String timeoutInMillis = String.valueOf(context.getProperty(CONNECTION_TIMEOUT).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS));
+ javaMailProperties.setProperty(propertyName, timeoutInMillis);
return javaMailProperties;
}
@@ -377,7 +372,7 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
}
processSession.getProvenanceReporter().receive(flowFile, this.displayUrl, "Received message from " + fromAddressesString, executionDuration);
- this.getLogger().info("Successfully received {} from {} in {} millis", new Object[] { flowFile, fromAddressesString, executionDuration });
+ this.getLogger().info("Successfully received {} from {} in {} millis", new Object[]{flowFile, fromAddressesString, executionDuration});
processSession.transfer(flowFile, REL_SUCCESS);
try {
http://git-wip-us.apache.org/repos/asf/nifi/blob/9c766894/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java
index 84a78f5..dabd881 100644
--- a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java
+++ b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java
@@ -98,6 +98,7 @@ public class ConsumeEmailTest {
runner.setProperty(ConsumeIMAP.FOLDER, "MYBOX");
runner.setProperty(ConsumeIMAP.USE_SSL, "false");
runner.setProperty(ConsumeIMAP.SHOULD_DELETE_MESSAGES, "false");
+ runner.setProperty(ConsumeIMAP.CONNECTION_TIMEOUT, "130 ms");
runner.run(2);
flowFiles = runner.getFlowFilesForRelationship(ConsumeIMAP.REL_SUCCESS);
@@ -108,11 +109,33 @@ public class ConsumeEmailTest {
ff.assertContentEquals("You've Got Mail - 1".getBytes(StandardCharsets.UTF_8));
}
+ @Test
+ public void validateConsumeIMAPWithTimeout() throws Exception {
+ TestRunner runner = TestRunners.newTestRunner(new TestImapProcessor(1));
+ runner.setProperty(ConsumeIMAP.HOST, "foo.bar.com");
+ runner.setProperty(ConsumeIMAP.PORT, "1234");
+ runner.setProperty(ConsumeIMAP.USER, "jon");
+ runner.setProperty(ConsumeIMAP.PASSWORD, "qhgwjgehr");
+ runner.setProperty(ConsumeIMAP.FOLDER, "MYBOX");
+ runner.setProperty(ConsumeIMAP.USE_SSL, "false");
+ runner.setProperty(ConsumeIMAP.SHOULD_DELETE_MESSAGES, "false");
+ runner.setProperty(ConsumeIMAP.CONNECTION_TIMEOUT, "${random():mod(10):plus(1)} secs");
+
+ runner.run(1);
+ List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ConsumeIMAP.REL_SUCCESS);
+ assertEquals(1, flowFiles.size());
+ MockFlowFile ff = flowFiles.get(0);
+ ff.assertContentEquals("You've Got Mail - 0".getBytes(StandardCharsets.UTF_8));
+ }
+
public static class TestImapProcessor extends ConsumeIMAP {
+
private final int messagesToGenerate;
+
TestImapProcessor(int messagesToGenerate) {
this.messagesToGenerate = messagesToGenerate;
}
+
@Override
protected ImapMailReceiver buildMessageReceiver(ProcessContext processContext) {
ImapMailReceiver receiver = mock(ImapMailReceiver.class);