You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2016/08/14 18:35:42 UTC

[1/2] nifi git commit: NIFI-2560 added connection timeout to IMAP/POP3

Repository: nifi
Updated Branches:
  refs/heads/master d3b96dcac -> 9c7668948


NIFI-2560 added connection timeout to IMAP/POP3


Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b7b56b1c
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b7b56b1c
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b7b56b1c

Branch: refs/heads/master
Commit: b7b56b1cade12489309ea6e8843336f82cde4b23
Parents: d3b96dc
Author: Oleg Zhurakousky <ol...@suitcase.io>
Authored: Fri Aug 12 16:20:03 2016 -0400
Committer: joewitt <jo...@apache.org>
Committed: Sun Aug 14 14:06:07 2016 -0400

----------------------------------------------------------------------
 .../email/AbstractEmailProcessor.java           | 22 ++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/nifi/blob/b7b56b1c/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
index e7fd27c..292d42b 100644
--- a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
+++ b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
@@ -46,6 +46,7 @@ import org.apache.nifi.processor.Relationship;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.io.OutputStreamCallback;
 import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.util.FormatUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.support.StaticListableBeanFactory;
@@ -122,6 +123,16 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
             .defaultValue("false")
             .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
             .build();
+    static final PropertyDescriptor CONNECTION_TIMEOUT = new PropertyDescriptor.Builder()
+            .name("connection.timeout")
+            .displayName("Connection timeout")
+            .description("The amount of time to wait to connect to Email server")
+            .required(true)
+            .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+            .expressionLanguageSupported(true)
+            .defaultValue("30 sec")
+            .build();
+
 
     static final Relationship REL_SUCCESS = new Relationship.Builder()
             .name("success")
@@ -144,6 +155,7 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
         SHARED_DESCRIPTORS.add(FOLDER);
         SHARED_DESCRIPTORS.add(FETCH_SIZE);
         SHARED_DESCRIPTORS.add(SHOULD_DELETE_MESSAGES);
+        SHARED_DESCRIPTORS.add(CONNECTION_TIMEOUT);
 
         SHARED_RELATIONSHIPS.add(REL_SUCCESS);
     }
@@ -287,17 +299,23 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
 
     /**
      * Extracts dynamic properties which typically represent the Java Mail
-     * properties from the {@link ProcessContext} returnining them as instance
+     * properties from the {@link ProcessContext} returning them as instance
      * of {@link Properties}
      */
     private Properties buildJavaMailProperties(ProcessContext context) {
         Properties javaMailProperties = new Properties();
         for (Entry<PropertyDescriptor, String> propertyDescriptorEntry : context.getProperties().entrySet()) {
-            if (propertyDescriptorEntry.getKey().isDynamic()) {
+            if (propertyDescriptorEntry.getKey().isDynamic()
+                    && !propertyDescriptorEntry.getKey().getName().equals("mail.imap.timeout")
+                    && !propertyDescriptorEntry.getKey().getName().equals("mail.pop3.timeout")) {
                 javaMailProperties.setProperty(propertyDescriptorEntry.getKey().getName(),
                         propertyDescriptorEntry.getValue());
             }
         }
+        String propertyName = this.getProtocol(context).equals("pop3") ? "mail.pop3.timeout" : "mail.imap.timeout";
+
+        javaMailProperties.setProperty(propertyName, String.valueOf(FormatUtils
+                .getTimeDuration(context.getProperty(CONNECTION_TIMEOUT).getValue().trim(), TimeUnit.MILLISECONDS)));
         return javaMailProperties;
     }
 


[2/2] nifi git commit: NIFI-2560 This closes #851. fixed property handling and added tests using it

Posted by jo...@apache.org.
NIFI-2560 This closes #851. fixed property handling and added tests using it


Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9c766894
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9c766894
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9c766894

Branch: refs/heads/master
Commit: 9c7668948c9a1541b9b317ab5352570910037d20
Parents: b7b56b1
Author: joewitt <jo...@apache.org>
Authored: Sun Aug 14 14:35:13 2016 -0400
Committer: joewitt <jo...@apache.org>
Committed: Sun Aug 14 14:35:25 2016 -0400

----------------------------------------------------------------------
 .../email/AbstractEmailProcessor.java           | 23 ++++++++------------
 .../nifi/processors/email/ConsumeEmailTest.java | 23 ++++++++++++++++++++
 2 files changed, 32 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/nifi/blob/9c766894/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
index 292d42b..62f4a88 100644
--- a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
+++ b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/main/java/org/apache/nifi/processors/email/AbstractEmailProcessor.java
@@ -46,7 +46,6 @@ import org.apache.nifi.processor.Relationship;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.io.OutputStreamCallback;
 import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.util.FormatUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.support.StaticListableBeanFactory;
@@ -58,8 +57,7 @@ import org.springframework.util.StreamUtils;
  * Base processor for implementing processors to consume messages from Email
  * servers using Spring Integration libraries.
  *
- * @param <T>
- *            the type of {@link AbstractMailReceiver}.
+ * @param <T> the type of {@link AbstractMailReceiver}.
  */
 abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends AbstractProcessor {
 
@@ -133,15 +131,14 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
             .defaultValue("30 sec")
             .build();
 
-
     static final Relationship REL_SUCCESS = new Relationship.Builder()
             .name("success")
             .description("All messages that are the are successfully received from Email server and converted to FlowFiles are routed to this relationship")
             .build();
 
-    static List<PropertyDescriptor> SHARED_DESCRIPTORS = new ArrayList<>();
+    final static List<PropertyDescriptor> SHARED_DESCRIPTORS = new ArrayList<>();
 
-    static Set<Relationship> SHARED_RELATIONSHIPS = new HashSet<>();
+    final static Set<Relationship> SHARED_RELATIONSHIPS = new HashSet<>();
 
     /*
      * Will ensure that list of PropertyDescriptors is build only once, since
@@ -219,8 +216,7 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
      * Delegates to sub-classes to build the target receiver as
      * {@link AbstractMailReceiver}
      *
-     * @param context
-     *            instance of {@link ProcessContext}
+     * @param context instance of {@link ProcessContext}
      * @return new instance of {@link AbstractMailReceiver}
      */
     protected abstract T buildMessageReceiver(ProcessContext context);
@@ -299,8 +295,8 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
 
     /**
      * Extracts dynamic properties which typically represent the Java Mail
-     * properties from the {@link ProcessContext} returning them as instance
-     * of {@link Properties}
+     * properties from the {@link ProcessContext} returning them as instance of
+     * {@link Properties}
      */
     private Properties buildJavaMailProperties(ProcessContext context) {
         Properties javaMailProperties = new Properties();
@@ -313,9 +309,8 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
             }
         }
         String propertyName = this.getProtocol(context).equals("pop3") ? "mail.pop3.timeout" : "mail.imap.timeout";
-
-        javaMailProperties.setProperty(propertyName, String.valueOf(FormatUtils
-                .getTimeDuration(context.getProperty(CONNECTION_TIMEOUT).getValue().trim(), TimeUnit.MILLISECONDS)));
+        final String timeoutInMillis = String.valueOf(context.getProperty(CONNECTION_TIMEOUT).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS));
+        javaMailProperties.setProperty(propertyName, timeoutInMillis);
         return javaMailProperties;
     }
 
@@ -377,7 +372,7 @@ abstract class AbstractEmailProcessor<T extends AbstractMailReceiver> extends Ab
         }
 
         processSession.getProvenanceReporter().receive(flowFile, this.displayUrl, "Received message from " + fromAddressesString, executionDuration);
-        this.getLogger().info("Successfully received {} from {} in {} millis", new Object[] { flowFile, fromAddressesString, executionDuration });
+        this.getLogger().info("Successfully received {} from {} in {} millis", new Object[]{flowFile, fromAddressesString, executionDuration});
         processSession.transfer(flowFile, REL_SUCCESS);
 
         try {

http://git-wip-us.apache.org/repos/asf/nifi/blob/9c766894/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java
index 84a78f5..dabd881 100644
--- a/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java
+++ b/nifi-nar-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/ConsumeEmailTest.java
@@ -98,6 +98,7 @@ public class ConsumeEmailTest {
         runner.setProperty(ConsumeIMAP.FOLDER, "MYBOX");
         runner.setProperty(ConsumeIMAP.USE_SSL, "false");
         runner.setProperty(ConsumeIMAP.SHOULD_DELETE_MESSAGES, "false");
+        runner.setProperty(ConsumeIMAP.CONNECTION_TIMEOUT, "130 ms");
 
         runner.run(2);
         flowFiles = runner.getFlowFilesForRelationship(ConsumeIMAP.REL_SUCCESS);
@@ -108,11 +109,33 @@ public class ConsumeEmailTest {
         ff.assertContentEquals("You've Got Mail - 1".getBytes(StandardCharsets.UTF_8));
     }
 
+    @Test
+    public void validateConsumeIMAPWithTimeout() throws Exception {
+        TestRunner runner = TestRunners.newTestRunner(new TestImapProcessor(1));
+        runner.setProperty(ConsumeIMAP.HOST, "foo.bar.com");
+        runner.setProperty(ConsumeIMAP.PORT, "1234");
+        runner.setProperty(ConsumeIMAP.USER, "jon");
+        runner.setProperty(ConsumeIMAP.PASSWORD, "qhgwjgehr");
+        runner.setProperty(ConsumeIMAP.FOLDER, "MYBOX");
+        runner.setProperty(ConsumeIMAP.USE_SSL, "false");
+        runner.setProperty(ConsumeIMAP.SHOULD_DELETE_MESSAGES, "false");
+        runner.setProperty(ConsumeIMAP.CONNECTION_TIMEOUT, "${random():mod(10):plus(1)} secs");
+
+        runner.run(1);
+        List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ConsumeIMAP.REL_SUCCESS);
+        assertEquals(1, flowFiles.size());
+        MockFlowFile ff = flowFiles.get(0);
+        ff.assertContentEquals("You've Got Mail - 0".getBytes(StandardCharsets.UTF_8));
+    }
+
     public static class TestImapProcessor extends ConsumeIMAP {
+
         private final int messagesToGenerate;
+
         TestImapProcessor(int messagesToGenerate) {
             this.messagesToGenerate = messagesToGenerate;
         }
+
         @Override
         protected ImapMailReceiver buildMessageReceiver(ProcessContext processContext) {
             ImapMailReceiver receiver = mock(ImapMailReceiver.class);