You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/08/12 23:44:35 UTC
[1/4] git commit: Merge pull request #9 from apache/master
Repository: incubator-streams
Updated Branches:
refs/heads/master d475b913d -> f0404ba8f
Merge pull request #9 from apache/master
Merge Apache
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/a22a99c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/a22a99c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/a22a99c4
Branch: refs/heads/master
Commit: a22a99c4d56de09d96714070d77448dad4a6bde6
Parents: 5c9a531 0103a96
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Tue Aug 5 15:01:59 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Tue Aug 5 15:01:59 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 1 +
.../ElasticsearchClientManager.java | 8 +-
.../ElasticsearchPersistReader.java | 21 +-
.../ElasticsearchPersistWriter.java | 4 +-
.../elasticsearch/ElasticsearchQuery.java | 98 +-
.../streams/mongo/MongoPersistWriter.java | 21 +-
.../regex/AbstractRegexExtensionExtractor.java | 23 +-
.../org/apache/streams/regex/RegexUtils.java | 43 +-
.../DatasiftDefaultActivitySerializer.java | 7 +-
.../DatasiftTweetActivitySerializer.java | 106 +--
.../main/jsonschema/com/datasift/Datasift.json | 181 +---
.../com/datasift/DatasiftTwitterUser.json | 61 ++
.../DatasiftActivitySerializerTest.java | 8 -
.../streams-provider-instagram/README.md | 17 +
.../InstagramMediaDataActivitySerializer.mup | 894 +++++++++++++++++++
.../metadata/instagram_to_activity_mapping.png | Bin 0 -> 1123684 bytes
.../streams-provider-instagram/pom.xml | 145 +++
.../instagram/InstagramConfigurator.java | 65 ++
.../processor/InstagramTypeConverter.java | 102 +++
.../provider/InstagramRecentMediaCollector.java | 165 ++++
.../provider/InstagramRecentMediaProvider.java | 115 +++
.../InstagramJsonActivitySerializer.java | 78 ++
.../serializer/util/InstagramActivityUtil.java | 285 ++++++
.../serializer/util/InstagramDeserializer.java | 33 +
.../com/instagram/InstagramConfiguration.json | 21 +
.../InstagramUserInformationConfiguration.json | 17 +
.../src/main/resources/reference.conf | 5 +
.../InstagramRecentMediaCollectorTest.java | 175 ++++
.../InstagramRecentMediaProviderTest.java | 160 ++++
.../test/InstagramActivitySerDeTest.java | 88 ++
.../src/test/resources/testMediaFeedObjects.txt | 2 +
.../src/test/resources/testtweets.txt | 695 ++++++++++++++
.../streams/rss/processor/RssTypeConverter.java | 72 ++
.../streams/rss/provider/RssEventProcessor.java | 8 +-
.../serializer/SyndEntryActivitySerializer.java | 198 +++-
.../rss/serializer/SyndEntrySerializer.java | 308 +++++++
.../streams/rss/test/RssTypeConverterTest.java | 31 +
.../test/SyndEntryActivitySerizlizerTest.java | 102 +++
.../streams/rss/test/Top100FeedsTest.java | 84 --
.../src/test/resources/TestSyndEntryJson.txt | 10 +
.../provider/SysomosHeartbeatStream.java | 10 +-
.../processor/TwitterEventProcessor.java | 46 +-
.../provider/TwitterStreamProcessor.java | 98 ++
.../twitter/provider/TwitterStreamProvider.java | 104 ++-
.../provider/TwitterStreamProviderTask.java | 71 --
.../local/builders/LocalStreamBuilder.java | 40 +-
.../local/tasks/StreamsProcessorTask.java | 35 +-
.../local/tasks/StreamsProviderTask.java | 2 +-
.../org/apache/streams/util/ComponentUtils.java | 32 +-
49 files changed, 4220 insertions(+), 675 deletions(-)
----------------------------------------------------------------------
[4/4] git commit: Merging PR#64 from robdouglas/STREAMS-139
Posted by mf...@apache.org.
Merging PR#64 from robdouglas/STREAMS-139
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/f0404ba8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/f0404ba8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/f0404ba8
Branch: refs/heads/master
Commit: f0404ba8f14d661e754bc809d2ff39bbac8e4a5a
Parents: d475b91 6eb9707
Author: mfranklin <mf...@apache.org>
Authored: Tue Aug 12 16:44:26 2014 -0500
Committer: mfranklin <mf...@apache.org>
Committed: Tue Aug 12 16:44:26 2014 -0500
----------------------------------------------------------------------
.../serializer/SyndEntryActivitySerializer.java | 31 ++++++++++++++++----
.../test/SyndEntryActivitySerizlizerTest.java | 27 +++++++++++++----
.../src/test/resources/TestSyndEntryJson.txt | 1 +
3 files changed, 48 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
[2/4] git commit: STREAMS-139 | Updated the buildProvider method in
the SyndEntry Serializer to ensure that we use a valid URI for the link
attribute
Posted by mf...@apache.org.
STREAMS-139 | Updated the buildProvider method in the SyndEntry Serializer to ensure that we use a valid URI for the link attribute
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/9e72ae74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/9e72ae74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/9e72ae74
Branch: refs/heads/master
Commit: 9e72ae74154eb942d3e12ab183cc25540a7bfbcc
Parents: a22a99c
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Tue Aug 5 16:29:40 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Tue Aug 5 16:29:40 2014 -0500
----------------------------------------------------------------------
.../serializer/SyndEntryActivitySerializer.java | 2 +-
.../test/SyndEntryActivitySerizlizerTest.java | 27 ++++++++++++++++----
.../src/test/resources/TestSyndEntryJson.txt | 1 +
3 files changed, 24 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9e72ae74/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
index ace10dc..9edcd12 100644
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
+++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
@@ -179,7 +179,7 @@ public class SyndEntryActivitySerializer implements ActivitySerializer<ObjectNod
uri = entry.get("uri").textValue();
if (uri != null) {
- if((uri.contains("http") || uri.contains("www")) || (link == null || !(link.contains("http") || link.contains("www")))) {
+ if((uri.startsWith("http") || uri.startsWith("www")) || (link == null || !(link.startsWith("http") || link.startsWith("www")))) {
resourceLocation = uri;
} else {
resourceLocation = link;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9e72ae74/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/SyndEntryActivitySerizlizerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/SyndEntryActivitySerizlizerTest.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/SyndEntryActivitySerizlizerTest.java
index 776f4ed..fd9a996 100644
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/SyndEntryActivitySerizlizerTest.java
+++ b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/SyndEntryActivitySerizlizerTest.java
@@ -31,10 +31,13 @@ import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.junit.Test;
+import java.net.URL;
import java.util.List;
import java.util.Scanner;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
public class SyndEntryActivitySerizlizerTest {
@@ -57,7 +60,7 @@ public class SyndEntryActivitySerizlizerTest {
activities.add(serializer.deserialize(node));
}
- assertEquals(10, activities.size());
+ assertEquals(11, activities.size());
for(int x = 0; x < activities.size(); x ++) {
ObjectNode n = objects.get(x);
@@ -66,9 +69,10 @@ public class SyndEntryActivitySerizlizerTest {
testActor(n.get("author").asText(), a.getActor());
testAuthor(n.get("author").asText(), a.getObject().getAuthor());
testProvider("id:providers:rss", "RSS", a.getProvider());
+ testProviderUrl(a.getProvider());
testVerb("post", a.getVerb());
testPublished(n.get("publishedDate").asText(), a.getPublished());
- testUrl(n.get("uri").asText(), a);
+ testUrl(n.get("uri").asText(), n.get("link").asText(), a);
}
}
@@ -95,8 +99,21 @@ public class SyndEntryActivitySerizlizerTest {
assertEquals(expectedDisplay, provider.getDisplayName());
}
- public void testUrl(String expected, Activity activity) {
- assertEquals(expected, activity.getUrl());
- assertEquals(expected, activity.getObject().getUrl());
+ public void testProviderUrl(Provider provider) {
+ URL url = null;
+
+ try {
+ url = new URL(provider.getUrl());
+ url.toURI();
+ } catch(Exception e) {
+ System.out.println("Threw an exception while trying to validate URL: " + provider.getUrl());
+ }
+
+ assertNotNull(url);
+ }
+
+ public void testUrl(String expectedURI, String expectedLink, Activity activity) {
+ assertTrue((expectedURI == activity.getUrl() || expectedLink == activity.getUrl()));
+ assertTrue((expectedURI == activity.getObject().getUrl() || expectedLink == activity.getObject().getUrl()));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9e72ae74/streams-contrib/streams-provider-rss/src/test/resources/TestSyndEntryJson.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/test/resources/TestSyndEntryJson.txt b/streams-contrib/streams-provider-rss/src/test/resources/TestSyndEntryJson.txt
index e559b3b..6fef187 100644
--- a/streams-contrib/streams-provider-rss/src/test/resources/TestSyndEntryJson.txt
+++ b/streams-contrib/streams-provider-rss/src/test/resources/TestSyndEntryJson.txt
@@ -8,3 +8,4 @@
{"author":"Greg Bates","categories":[],"contents":[{}],"description":{"value":"<a href=\"http://www.programmableweb.com/api/change.org\"><img class=\"imgRight\" src=\"http://www.programmableweb.com/images/apis/at8881.png\" alt=\"Change.org\" /></a>HackSummit features a partnership with The Evolve Organisation focused on two problems for democracy. Kings of Code Hack Battle's brilliant hacks. Plus: Yodlee links incubators for fintech innovation, LinkedIn launches Sponsored Updates API, and 10 new APIs.","type":"text/html"},"link":"http://feedproxy.google.com/~r/ProgrammableWeb/~3/12JNO5FUkCE/","modules":["http://purl.org/dc/elements/1.1/"],"publishedDate":"2014-04-28T16:01:00.000-05:00","title":"Today in APIs: HackSummit Partners with Evolve, and 10 New APIs","uri":"http://blog.programmableweb.com/?p=83441"}
{"author":"Guest Author","categories":[],"contents":[{}],"description":{"value":"<img src=\"http://blog.programmableweb.com/wp-content/Screen-Shot-2014-03-07-at-4.51.06-PM5-150x58.png\" alt=\"\" width=\"150\" height=\"58\" align=\"right\" />So you’ve built a stellar API. Now what? How do you get the word out? First, throw everything you know about marketing out the window. Indeed, the old style of marketing doesn’t apply when it comes to APIs. The notion of building a marketing campaign replete with ads and PR fluff is virtually non-existent in the world of API marketing. Why? When you create an API, you are creating a community. This community of users will drive your marketing in a highly decentralized way.","type":"text/html"},"link":"http://feedproxy.google.com/~r/ProgrammableWeb/~3/O2Ol3kBFpqk/","modules":["http://purl.org/dc/elements/1.1/"],"publishedDate":"2014-04-28T15:00:07.000-05:00","title":"10 Ways To Market Your API (And Why Old-School Marketing Won’t Work)","uri"
:"http://blog.programmableweb.com/?p=83440"}
{"author":"Michael Vizard","categories":[],"contents":[{}],"description":{"value":"<img src=\"http://blog.programmableweb.com/wp-content/Screen-Shot-2014-04-28-at-2.25.35-PM-150x61.png\" alt=\"\" width=\"150\" height=\"61\" align=\"right\" />As a first step toward enabling new economic models for software consumption, <a href=\"http://www-03.ibm.com/press/us/en/pressrelease/43723.wss\">today IBM opened an IBM Cloud marketplace</a>, through which it and its partners will make software available to enterprise IT customers. In addition, IBM also announced that it is launching BlueMix Garages.","type":"text/html"},"link":"http://feedproxy.google.com/~r/ProgrammableWeb/~3/-Fot4vkTNi0/","modules":["http://purl.org/dc/elements/1.1/"],"publishedDate":"2014-04-28T13:26:34.000-05:00","title":"IBM Opens Marketplace to Help Drive Emerging API Economy","uri":"http://blog.programmableweb.com/?p=83495"}
+{"author":"Michael Vizard","categories":[],"contents":[{}],"description":{"value":"<img src=\"http://blog.programmableweb.com/wp-content/Screen-Shot-2014-04-28-at-2.25.35-PM-150x61.png\" alt=\"\" width=\"150\" height=\"61\" align=\"right\" />As a first step toward enabling new economic models for software consumption, <a href=\"http://www-03.ibm.com/press/us/en/pressrelease/43723.wss\">today IBM opened an IBM Cloud marketplace</a>, through which it and its partners will make software available to enterprise IT customers. In addition, IBM also announced that it is launching BlueMix Garages.","type":"text/html"},"link":"http://www.whitehouse.gov/blog/2014/08/05/john-podesta-cost-inaction","modules":["http://purl.org/dc/elements/1.1/"],"publishedDate":"2014-04-28T13:26:34.000-05:00","title":"IBM Opens Marketplace to Help Drive Emerging API Economy","uri":"292776 at http://www.whitehouse.gov"}
\ No newline at end of file
[3/4] git commit: STREAMS-139 | Modified buildProvider to make logic
more understandable
Posted by mf...@apache.org.
STREAMS-139 | Modified buildProvider to make logic more understandable
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/6eb9707a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/6eb9707a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/6eb9707a
Branch: refs/heads/master
Commit: 6eb9707ae9de40e399f3a89801fa952867a5c9a8
Parents: 9e72ae7
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Wed Aug 6 11:14:55 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Wed Aug 6 11:14:55 2014 -0500
----------------------------------------------------------------------
.../serializer/SyndEntryActivitySerializer.java | 31 ++++++++++++++++----
1 file changed, 25 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6eb9707a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
index 9edcd12..06839f3 100644
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
+++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
@@ -178,12 +178,20 @@ public class SyndEntryActivitySerializer implements ActivitySerializer<ObjectNod
if (entry.get("uri") != null)
uri = entry.get("uri").textValue();
- if (uri != null) {
- if((uri.startsWith("http") || uri.startsWith("www")) || (link == null || !(link.startsWith("http") || link.startsWith("www")))) {
- resourceLocation = uri;
- } else {
- resourceLocation = link;
- }
+ /**
+ * Order of precedence for resourceLocation selection
+ *
+ * 1. Valid URI
+ * 2. Valid Link
+ * 3. Non-null URI
+ * 4. Non-null Link
+ */
+ if(isValidResource(uri))
+ resourceLocation = uri;
+ else if(isValidResource(link))
+ resourceLocation = link;
+ else if(uri != null || link != null) {
+ resourceLocation = (uri != null) ? uri : link;
}
provider.setId("id:providers:rss");
@@ -194,6 +202,17 @@ public class SyndEntryActivitySerializer implements ActivitySerializer<ObjectNod
}
/**
+ * Tests whether or not the passed in resource is a valid URI
+ * @param resource
+ * @return boolean of whether or not the resource is valid
+ */
+ private boolean isValidResource(String resource) {
+ if(resource != null && resource.startsWith("http") || resource.startsWith("www"))
+ return true;
+ return false;
+ }
+
+ /**
* Given an RSS object and an existing activity,
* add the Rome extension to that activity and return it
*