You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/12/13 19:00:14 UTC
[2/4] incubator-streams git commit: STREAMS-469: fit failing ITs in
streams-provider-rss
STREAMS-469: fit failing ITs in streams-provider-rss
As best I can tell, junit forking was allowing the tests to pass - they started failing under TestNG.
Wound up removing the static modifier from some state where it didn\u2019t really make sense.
Along the way, upgraded the underlying library to one under active maintenance.
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/21086e9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/21086e9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/21086e9a
Branch: refs/heads/master
Commit: 21086e9a84721ef0d777a65a9cbd9e1f28040b18
Parents: 899bb84
Author: Steve Blackmon <sb...@apache.org>
Authored: Sun Dec 11 21:35:05 2016 -0600
Committer: Steve Blackmon <sb...@apache.org>
Committed: Sun Dec 11 21:35:05 2016 -0600
----------------------------------------------------------------------
streams-contrib/streams-provider-rss/pom.xml | 4 +-
.../streams/rss/provider/RssEventProcessor.java | 3 +-
.../rss/provider/RssStreamProviderTask.java | 11 ++--
.../rss/serializer/SyndEntrySerializer.java | 39 +++++++-------
.../rss/provider/RssStreamProviderTaskIT.java | 55 ++++++++------------
5 files changed, 51 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/pom.xml b/streams-contrib/streams-provider-rss/pom.xml
index 5cf8b79..6a237d5 100644
--- a/streams-contrib/streams-provider-rss/pom.xml
+++ b/streams-contrib/streams-provider-rss/pom.xml
@@ -86,9 +86,9 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>rome</groupId>
+ <groupId>com.rometools</groupId>
<artifactId>rome</artifactId>
- <version>1.0</version>
+ <version>1.7.0</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java
index 078356c..cf768e0 100644
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java
+++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java
@@ -24,7 +24,8 @@ import org.apache.streams.rss.serializer.SyndEntryActivitySerializer;
import org.apache.streams.rss.serializer.SyndEntrySerializer;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.sun.syndication.feed.synd.SyndEntry;
+import com.rometools.rome.feed.synd.SyndEntry;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
index 03a66d1..8437ee0 100644
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
+++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
@@ -25,10 +25,11 @@ import org.apache.streams.rss.serializer.SyndEntrySerializer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.annotations.VisibleForTesting;
-import com.sun.syndication.feed.synd.SyndEntry;
-import com.sun.syndication.feed.synd.SyndFeed;
-import com.sun.syndication.io.FeedException;
-import com.sun.syndication.io.SyndFeedInput;
+import com.rometools.rome.feed.synd.SyndEntry;
+import com.rometools.rome.feed.synd.SyndFeed;
+import com.rometools.rome.io.FeedException;
+import com.rometools.rome.io.SyndFeedInput;
+
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,7 +78,7 @@ public class RssStreamProviderTask implements Runnable {
* Map that contains the Set of previously seen articles by an rss feed.
*/
@VisibleForTesting
- protected static final Map<String, Set<String>> PREVIOUSLY_SEEN = new ConcurrentHashMap<>();
+ protected Map<String, Set<String>> PREVIOUSLY_SEEN = new ConcurrentHashMap<>();
private BlockingQueue<StreamsDatum> dataQueue;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java
index 63920e7..1b84d0a 100644
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java
+++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java
@@ -21,16 +21,15 @@ package org.apache.streams.rss.serializer;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.sun.syndication.feed.module.Module;
-import com.sun.syndication.feed.rss.Category;
-import com.sun.syndication.feed.rss.Content;
-import com.sun.syndication.feed.rss.Enclosure;
-import com.sun.syndication.feed.synd.SyndContent;
-import com.sun.syndication.feed.synd.SyndEnclosure;
-import com.sun.syndication.feed.synd.SyndEntry;
-import com.sun.syndication.feed.synd.SyndFeed;
-import com.sun.syndication.feed.synd.SyndImage;
-import com.sun.syndication.feed.synd.SyndLinkImpl;
+import com.rometools.rome.feed.module.Module;
+import com.rometools.rome.feed.rss.Enclosure;
+import com.rometools.rome.feed.synd.SyndContent;
+import com.rometools.rome.feed.synd.SyndEnclosure;
+import com.rometools.rome.feed.synd.SyndEntry;
+import com.rometools.rome.feed.synd.SyndFeed;
+import com.rometools.rome.feed.synd.SyndImage;
+import com.rometools.rome.feed.synd.SyndLinkImpl;
+
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import org.slf4j.Logger;
@@ -82,9 +81,9 @@ public class SyndEntrySerializer {
}
ArrayNode cats = factory.arrayNode();
for (Object obj : categories) {
- if (obj instanceof Category) {
+ if (obj instanceof com.rometools.rome.feed.rss.Category) {
ObjectNode catNode = factory.objectNode();
- Category category = (Category) obj;
+ com.rometools.rome.feed.rss.Category category = (com.rometools.rome.feed.rss.Category) obj;
if (category.getDomain() != null) {
catNode.put("domain", category.getDomain());
}
@@ -92,8 +91,8 @@ public class SyndEntrySerializer {
catNode.put("value", category.getValue());
}
cats.add(catNode);
- } else if (obj instanceof com.sun.syndication.feed.atom.Category) {
- com.sun.syndication.feed.atom.Category category = (com.sun.syndication.feed.atom.Category) obj;
+ } else if (obj instanceof com.rometools.rome.feed.atom.Category) {
+ com.rometools.rome.feed.atom.Category category = (com.rometools.rome.feed.atom.Category) obj;
ObjectNode catNode = factory.objectNode();
if (category.getLabel() != null) {
catNode.put("label", category.getLabel());
@@ -120,13 +119,13 @@ public class SyndEntrySerializer {
ArrayNode contentsArray = factory.arrayNode();
for (Object obj : contents) {
ObjectNode content = factory.objectNode();
- if (obj instanceof Content) {
- Content rssContent = (Content) obj;
+ if (obj instanceof com.rometools.rome.feed.rss.Content) {
+ com.rometools.rome.feed.rss.Content rssContent = (com.rometools.rome.feed.rss.Content) obj;
content.put("type", rssContent.getType());
content.put("value", rssContent.getValue());
}
- if (obj instanceof com.sun.syndication.feed.atom.Content) {
- com.sun.syndication.feed.atom.Content atomContent = (com.sun.syndication.feed.atom.Content) obj;
+ if (obj instanceof com.rometools.rome.feed.atom.Content) {
+ com.rometools.rome.feed.atom.Content atomContent = (com.rometools.rome.feed.atom.Content) obj;
content.put("type", atomContent.getType());
content.put("value", atomContent.getValue());
content.put("mode", atomContent.getMode());
@@ -248,7 +247,9 @@ public class SyndEntrySerializer {
}
private void serializeLinks(ObjectNode root, JsonNodeFactory factory, List links) {
- if (links.get(0) instanceof String) {
+ if( links.size() == 0 ) {
+ root.put("links", factory.arrayNode());
+ } else if (links.get(0) instanceof String) {
serializeListOfStrings(links, "links", root, factory);
} else if (links.get(0) instanceof SyndLinkImpl) {
ArrayNode linksArray = factory.arrayNode();
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java
index 87ff8fc..564e215 100644
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java
+++ b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java
@@ -22,7 +22,6 @@ import org.apache.streams.core.StreamsDatum;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
-import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.net.URL;
@@ -39,18 +38,6 @@ import static org.testng.Assert.assertTrue;
public class RssStreamProviderTaskIT {
/**
- * clearPreviouslySeen.
- */
- @BeforeClass
- public void clearPreviouslySeen() {
- //some test runners run in parallel so needs to be synchronized
- //if tests are run in parallel test will have undetermined results.
- synchronized (RssStreamProviderTask.PREVIOUSLY_SEEN) {
- RssStreamProviderTask.PREVIOUSLY_SEEN.clear();
- }
- }
-
- /**
* Test that a task can read a valid rss from a url and queue the data.
* @throws Exception Exception
*/
@@ -61,7 +48,7 @@ public class RssStreamProviderTaskIT {
RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url");
Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
assertEquals(batch.size(), queue.size(), "Expected batch size to be the same as amount of queued datums");
- RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ task.PREVIOUSLY_SEEN.put("fake url", batch);
//Test that it will out previously seen articles
queue.clear();
batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
@@ -76,22 +63,22 @@ public class RssStreamProviderTaskIT {
public void testPerpetualNoTimeFramePull() throws Exception {
com.healthmarketscience.common.util.resource.Handler.init();
BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
- RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", new DateTime().minusYears(5), 10000, true);
+ RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", new DateTime().minusYears(10), 10000, true);
Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
assertEquals(batch.size(), queue.size(), "Expected batch size to be the same as amount of queued datums");
- RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ task.PREVIOUSLY_SEEN.put("fake url", batch);
//Test that it will not out previously seen articles
queue.clear();
batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals(0, queue.size(), "Expected queue size to be 0");
- assertEquals(20, batch.size(), "Expected batch size to be 20");
- RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ assertEquals( queue.size(), 0 );
+ assertEquals( batch.size(), 20 );
+ task.PREVIOUSLY_SEEN.put("fake url", batch);
//Test that not seen urls aren't blocked.
queue.clear();
batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist2.xml"));
assertEquals(batch.size(), queue.size());
- assertEquals(25, queue.size(), "Expected queue size to be 25");
- assertEquals(25, batch.size(), "Expected batch size to be 25");
+ assertEquals( queue.size(), 25);
+ assertEquals( batch.size(), 25);
}
/**
@@ -105,15 +92,15 @@ public class RssStreamProviderTaskIT {
DateTime publishedSince = new DateTime().withYear(2014).withDayOfMonth(5).withMonthOfYear(9).withZone(DateTimeZone.UTC);
RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", publishedSince, 10000, false);
Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals( 15, queue.size());
- assertEquals( 20 , batch.size());
+ assertEquals( queue.size(), 15);
+ assertEquals( batch.size(), 20);
assertTrue( queue.size() < batch.size());
- RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ task.PREVIOUSLY_SEEN.put("fake url", batch);
//Test that it will out previously seen articles
queue.clear();
batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals( 15, queue.size());
- assertEquals( 20 , batch.size());
+ assertEquals( queue.size(), 15);
+ assertEquals( batch.size(), 20);
assertTrue( queue.size() < batch.size());
}
@@ -128,22 +115,22 @@ public class RssStreamProviderTaskIT {
DateTime publishedSince = new DateTime().withYear(2014).withDayOfMonth(5).withMonthOfYear(9).withZone(DateTimeZone.UTC);
RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", publishedSince, 10000, true);
Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals( 15, queue.size());
- assertEquals( 20 , batch.size());
+ assertEquals( queue.size(),15);
+ assertEquals( batch.size(), 20);
assertTrue( queue.size() < batch.size());
- RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ task.PREVIOUSLY_SEEN.put("fake url", batch);
//Test that it will not out put previously seen articles
queue.clear();
batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals( 0, queue.size());
- assertEquals( 20 , batch.size());
+ assertEquals( queue.size(), 0);
+ assertEquals( batch.size(), 20);
assertTrue( queue.size() < batch.size());
- RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ task.PREVIOUSLY_SEEN.put("fake url", batch);
batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist2.xml"));
assertTrue( queue.size() < batch.size());
- assertEquals(3, queue.size(), "Expected queue size to be 0");
- assertEquals(25, batch.size(), "Expected batch size to be 0");
+ assertEquals(queue.size(), 3);
+ assertEquals(batch.size(), 25);
}