You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/05/05 20:54:29 UTC
[42/52] [abbrv] git commit: Added types for TwitterEvent,
TwitterFriendList Writing tests for processor-json
Added types for TwitterEvent, TwitterFriendList
Writing tests for processor-json
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/b0a71691
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/b0a71691
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/b0a71691
Branch: refs/heads/sblackmon
Commit: b0a7169194289bb429ef086d1ad6375b551abfb8
Parents: 53bce53
Author: sblackmon <sb...@w2odigital.com>
Authored: Mon Apr 14 14:16:45 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Mon Apr 14 14:16:45 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 1 +
streams-contrib/streams-provider-twitter/pom.xml | 10 ++--------
.../streams/twitter/provider/TwitterEventClassifier.java | 9 ++++++---
.../streams/twitter/provider/TwitterStreamProvider.java | 1 +
.../twitter/serializer/TwitterJsonActivitySerializer.java | 4 ++++
5 files changed, 14 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b0a71691/streams-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
index d80fc63..d6873bb 100644
--- a/streams-contrib/pom.xml
+++ b/streams-contrib/pom.xml
@@ -46,6 +46,7 @@
<module>streams-persist-mongo</module>
<!--<module>streams-processor-lucene</module>-->
<!--<module>streams-processor-tika</module>-->
+ <module>streams-processor-json</module>
<module>streams-processor-urls</module>
<module>streams-provider-datasift</module>
<module>streams-provider-facebook</module>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b0a71691/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
index 3c27b8c..bf25631 100644
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@ -58,14 +58,6 @@
<artifactId>guava</artifactId>
</dependency>
<dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- </dependency>
- <dependency>
<groupId>com.twitter</groupId>
<artifactId>hbc-core</artifactId>
<version>2.0.0</version>
@@ -120,6 +112,8 @@
<sourcePaths>
<sourcePath>src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json</sourcePath>
<sourcePath>src/main/jsonschema/com/twitter/Delete.json</sourcePath>
+ <sourcePath>src/main/jsonschema/com/twitter/UserstreamEvent.json</sourcePath>
+ <sourcePath>src/main/jsonschema/com/twitter/FriendList.json</sourcePath>
<sourcePath>src/main/jsonschema/com/twitter/Retweet.json</sourcePath>
<sourcePath>src/main/jsonschema/com/twitter/tweet.json</sourcePath>
</sourcePaths>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b0a71691/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
index b577e42..2ed73bb 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
@@ -4,9 +4,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Preconditions;
import com.jayway.jsonassert.JsonAssert;
import org.apache.commons.lang.StringUtils;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
+import org.apache.streams.twitter.pojo.*;
import org.apache.streams.twitter.serializer.StreamsTwitterMapper;
import java.io.IOException;
@@ -47,6 +45,11 @@ public class TwitterEventClassifier {
return Retweet.class;
else if( objectNode.findValue("delete") != null )
return Delete.class;
+ else if( objectNode.findValue("friends") != null ||
+ objectNode.findValue("friends_str") != null )
+ return FriendList.class;
+ else if( objectNode.findValue("target_object") != null )
+ return UserstreamEvent.class;
else
return Tweet.class;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b0a71691/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
index ba88803..890fdd4 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
@@ -133,6 +133,7 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable, Dat
userstreamEndpoint.withFollowings(true);
userstreamEndpoint.withUser(false);
userstreamEndpoint.allReplies(false);
+ userstreamEndpoint.addPostParameter("stringify_friend_ids", "true");
endpoint = userstreamEndpoint;
}
else if(config.getEndpoint().equals("sample") ) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b0a71691/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
index bfceae0..8f7b25e 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
@@ -22,6 +22,7 @@ import org.apache.streams.jackson.StreamsJacksonModule;
import org.apache.streams.pojo.json.Activity;
import org.apache.streams.pojo.json.Provider;
import org.apache.streams.twitter.pojo.Delete;
+import org.apache.streams.twitter.pojo.UserstreamEvent;
import org.apache.streams.twitter.pojo.Retweet;
import org.apache.streams.twitter.pojo.Tweet;
import org.apache.streams.twitter.provider.TwitterEventClassifier;
@@ -47,6 +48,7 @@ public class TwitterJsonActivitySerializer implements ActivitySerializer<String>
TwitterJsonTweetActivitySerializer tweetActivitySerializer = new TwitterJsonTweetActivitySerializer();
TwitterJsonRetweetActivitySerializer retweetActivitySerializer = new TwitterJsonRetweetActivitySerializer();
TwitterJsonDeleteActivitySerializer deleteActivitySerializer = new TwitterJsonDeleteActivitySerializer();
+ TwitterJsonUserstreameventActivitySerializer userstreameventActivitySerializer = new TwitterJsonUserstreameventActivitySerializer();
@Override
public String serializationFormat() {
@@ -70,6 +72,8 @@ public class TwitterJsonActivitySerializer implements ActivitySerializer<String>
activity = retweetActivitySerializer.deserialize(serialized);
else if( documentSubType == Delete.class )
activity = deleteActivitySerializer.deserialize(serialized);
+ else if( documentSubType == UserstreamEvent.class )
+ activity = userstreameventActivitySerializer.deserialize(serialized);
else throw new ActivitySerializerException("unrecognized type");
return activity;