You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/05/27 18:37:27 UTC
[1/8] git commit: STREAMS-93 | Updated the Hashtag POJO schema in
tweet.json,
added dependency on streams-processor-urls in streams-provider-twitter (to get
LinkDetails POJO) and updated all Twitter serializers to add in generic
extensions
Repository: incubator-streams
Updated Branches:
refs/heads/master 4580d533d -> 0fc460695
STREAMS-93 | Updated the Hashtag POJO schema in tweet.json, added dependency on streams-processor-urls in streams-provider-twitter (to get LinkDetails POJO) and updated all Twitter serializers to add in generic extensions
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e29eadc9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e29eadc9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e29eadc9
Branch: refs/heads/master
Commit: e29eadc9c29bb0dbb1dc2115804e84823d5674b2
Parents: aa376fd
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Thu May 22 11:04:03 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Thu May 22 11:04:03 2014 -0500
----------------------------------------------------------------------
.../streams-provider-twitter/pom.xml | 5 +
.../TwitterJsonActivitySerializer.java | 56 +++++++++-
.../TwitterJsonRetweetActivitySerializer.java | 101 +++++++++++++------
.../TwitterJsonTweetActivitySerializer.java | 90 ++++++++++++-----
.../src/main/jsonschema/com/twitter/tweet.json | 21 +++-
5 files changed, 209 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e29eadc9/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
index c104810..4bd4240 100644
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@ -46,6 +46,11 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
+ <artifactId>streams-processor-urls</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
<artifactId>streams-config</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e29eadc9/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
index 0ab3448..766aaa6 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
@@ -13,6 +13,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector;
import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import org.apache.commons.lang.NotImplementedException;
import org.apache.streams.data.ActivitySerializer;
@@ -20,21 +21,26 @@ import org.apache.streams.exceptions.ActivitySerializerException;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.jackson.StreamsJacksonModule;
import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
import org.apache.streams.pojo.json.Provider;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
+import org.apache.streams.twitter.Url;
+import org.apache.streams.twitter.pojo.*;
import org.apache.streams.twitter.provider.TwitterEventClassifier;
+import org.apache.streams.urls.LinkDetails;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import java.io.IOException;
-import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.io.Serializable;
import java.util.Map;
+import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
+
/**
* Created by sblackmon on 3/26/14.
*/
@@ -81,6 +87,48 @@ public class TwitterJsonActivitySerializer implements ActivitySerializer<String>
throw new NotImplementedException();
}
+ public static Actor buildActorRetweet(Retweet tweet) {
+ return buildActor(tweet.getUser());
+ }
+
+ public static Actor buildActorTweet(Tweet tweet) {
+ return buildActor(tweet.getUser());
+ }
+
+ public static Actor buildActor(User user) {
+ Actor actor = new Actor();
+
+ actor.setId(formatId(
+ Optional.fromNullable(
+ user.getIdStr())
+ .or(Optional.of(user.getId().toString()))
+ .orNull()
+ ));
+ actor.setDisplayName(user.getScreenName());
+ if (user.getUrl()!=null){
+ actor.setUrl(user.getUrl());
+ }
+
+ actor.setDisplayName(user.getName());
+ actor.setSummary(user.getDescription());
+
+ Map<String, Object> extensions = new HashMap<String, Object>();
+ extensions.put("location", user.getLocation());
+ extensions.put("posts", user.getStatusesCount());
+ extensions.put("favorites", user.getFavouritesCount());
+ extensions.put("followers", user.getFollowersCount());
+
+ Map<String, Object> image = new HashMap<String, Object>();
+ image.put("url", user.getProfileImageUrlHttps());
+
+ extensions.put("image", image);
+ extensions.put("screenName", user.getScreenName());
+
+ actor.setAdditionalProperty("extensions", extensions);
+
+ return actor;
+ }
+
public static Provider getProvider() {
Provider provider = new Provider();
provider.setId("id:providers:twitter");
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e29eadc9/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
index f8ade3f..da086a6 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
@@ -12,13 +12,15 @@ import org.apache.streams.pojo.json.Activity;
import org.apache.streams.pojo.json.ActivityObject;
import org.apache.streams.pojo.json.Actor;
import org.apache.streams.twitter.Url;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
+import org.apache.streams.twitter.pojo.*;
+import org.apache.streams.urls.LinkDetails;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.io.Serializable;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.io.Serializable;
import java.util.List;
import java.util.Map;
@@ -26,18 +28,20 @@ import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerialize
import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
/**
-* Created with IntelliJ IDEA.
-* User: mdelaet
-* Date: 9/30/13
-* Time: 9:24 AM
-* To change this template use File | Settings | File Templates.
-*/
+ * Created with IntelliJ IDEA.
+ * User: mdelaet
+ * Date: 9/30/13
+ * Time: 9:24 AM
+ * To change this template use File | Settings | File Templates.
+ */
public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<String>, Serializable {
public TwitterJsonRetweetActivitySerializer() {
}
+ private final static Logger LOGGER = LoggerFactory.getLogger(TwitterJsonRetweetActivitySerializer.class);
+
@Override
public String serializationFormat() {
return null;
@@ -62,10 +66,12 @@ public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<
}
Activity activity = new Activity();
- activity.setActor(buildActor(retweet));
+ activity.setActor(buildActorRetweet(retweet));
activity.setVerb("share");
+ Tweet retweetStatus = retweet.getRetweetedStatus();
+
if( retweet.getRetweetedStatus() != null )
- activity.setObject(buildActivityObject(retweet.getRetweetedStatus()));
+ activity.setObject(buildActivityObject(retweetStatus));
activity.setId(TwitterJsonActivitySerializer.formatId(activity.getVerb(),
Optional.fromNullable(
retweet.getIdStr())
@@ -88,34 +94,71 @@ public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<
} catch( Exception e ) {
throw new ActivitySerializerException("Unable to determine content", e);
}
- activity.setUrl("http://twitter.com/" + retweet.getIdStr());
+ activity.setUrl("http://twitter.com/" + retweet.getUser().getIdStr() + "/status/" + retweet.getIdStr());
activity.setLinks(TwitterJsonTweetActivitySerializer.getLinks(retweet.getRetweetedStatus()));
+
+ addTwitterExtensions(activity, retweetStatus);
addTwitterExtension(activity, mapper.convertValue(retweet, ObjectNode.class));
addLocationExtension(activity, retweet);
+
return activity;
}
+ public static void addTwitterExtensions(Activity activity, Tweet tweet) {
+ Map<String, Object> extensions = ensureExtensions(activity);
+
+ List<String> hashtags = new ArrayList<String>();
+ for(Hashtag hashtag : tweet.getEntities().getHashtags()) {
+ hashtags.add(hashtag.getText());
+ }
+ extensions.put("hashtags", hashtags);
+
+ Map<String, Object> likes = new HashMap<String, Object>();
+ likes.put("perspectival", tweet.getFavorited());
+ likes.put("count", tweet.getAdditionalProperties().get("favorite_count"));
+
+ extensions.put("likes", likes);
+
+ Map<String, Object> rebroadcasts = new HashMap<String, Object>();
+ rebroadcasts.put("perspectival", tweet.getRetweeted());
+ rebroadcasts.put("count", tweet.getRetweetCount());
+
+ extensions.put("rebroadcasts", rebroadcasts);
+
+ List<Map<String, Object>> userMentions = new ArrayList<Map<String, Object>>();
+ Entities entities = tweet.getEntities();
+
+ for(UserMentions user : entities.getUserMentions()) {
+ //Map the twitter user object into an actor
+ Map<String, Object> actor = new HashMap<String, Object>();
+ actor.put("id", "id:twitter:" + user.getIdStr());
+ actor.put("displayName", user.getScreenName());
+
+ userMentions.add(actor);
+ }
+
+ extensions.put("user_mentions", userMentions);
+
+ List<LinkDetails> urls = new ArrayList<LinkDetails>();
+ for(Url url : entities.getUrls()) {
+ LinkDetails linkDetails = new LinkDetails();
+
+ linkDetails.setFinalURL(url.getExpandedUrl());
+ linkDetails.setNormalizedURL(url.getDisplayUrl());
+ linkDetails.setOriginalURL(url.getUrl());
+
+ urls.add(linkDetails);
+ }
+ extensions.put("urls", urls);
+
+ extensions.put("keywords", tweet.getText());
+ }
+
@Override
public List<Activity> deserializeAll(List<String> serializedList) {
return null;
}
- public static Actor buildActor(Retweet retweet) {
- Actor actor = new Actor();
- User user = retweet.getUser();
- actor.setId(formatId(
- Optional.fromNullable(
- user.getIdStr())
- .or(Optional.of(user.getId().toString()))
- .orNull()
- ));
- actor.setDisplayName(user.getScreenName());
- if (user.getUrl()!=null){
- actor.setUrl(user.getUrl());
- }
- return actor;
- }
-
public static ActivityObject buildActivityObject(Tweet tweet) {
ActivityObject actObj = new ActivityObject();
String id = Optional.fromNullable(
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e29eadc9/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
index 1d00b7a..b8de327 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
@@ -14,11 +14,13 @@ import org.apache.streams.pojo.json.Activity;
import org.apache.streams.pojo.json.ActivityObject;
import org.apache.streams.pojo.json.Actor;
import org.apache.streams.twitter.Url;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
+import org.apache.streams.twitter.pojo.*;
+import org.apache.streams.urls.LinkDetails;
+import twitter4j.HashtagEntity;
import java.io.IOException;
import java.io.Serializable;
+import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -27,12 +29,12 @@ import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerialize
import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
/**
-* Created with IntelliJ IDEA.
-* User: mdelaet
-* Date: 9/30/13
-* Time: 9:24 AM
-* To change this template use File | Settings | File Templates.
-*/
+ * Created with IntelliJ IDEA.
+ * User: mdelaet
+ * Date: 9/30/13
+ * Time: 9:24 AM
+ * To change this template use File | Settings | File Templates.
+ */
public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<String>, Serializable {
@Override
@@ -60,7 +62,7 @@ public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<St
Activity activity = new Activity();
- activity.setActor(buildActor(tweet));
+ activity.setActor(buildActorTweet(tweet));
activity.setVerb("post");
activity.setId(formatId(activity.getVerb(),
Optional.fromNullable(
@@ -78,35 +80,71 @@ public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<St
activity.setProvider(getProvider());
activity.setTitle("");
activity.setContent(tweet.getText());
- activity.setUrl("http://twitter.com/" + tweet.getIdStr());
+ activity.setUrl("http://twitter.com/" + tweet.getUser().getIdStr() + "/status/" + tweet.getIdStr());
activity.setLinks(getLinks(tweet));
addTwitterExtension(activity, mapper.convertValue(tweet, ObjectNode.class));
addLocationExtension(activity, tweet);
+ addTwitterExtensions(activity, tweet);
+
return activity;
}
+ public static void addTwitterExtensions(Activity activity, Tweet tweet) {
+ Map<String, Object> extensions = ensureExtensions(activity);
+
+ List<String> hashtags = new ArrayList<String>();
+ for(Hashtag hashtag : tweet.getEntities().getHashtags()) {
+ hashtags.add(hashtag.getText());
+ }
+ extensions.put("hashtags", hashtags);
+
+ Map<String, Object> likes = new HashMap<String, Object>();
+ likes.put("perspectival", tweet.getFavorited());
+ likes.put("count", tweet.getAdditionalProperties().get("favorite_count"));
+
+ extensions.put("likes", likes);
+
+ Map<String, Object> rebroadcasts = new HashMap<String, Object>();
+ rebroadcasts.put("perspectival", tweet.getRetweeted());
+ rebroadcasts.put("count", tweet.getRetweetCount());
+
+ extensions.put("rebroadcasts", rebroadcasts);
+
+ List<Map<String, Object>> userMentions = new ArrayList<Map<String, Object>>();
+ Entities entities = tweet.getEntities();
+
+ for(UserMentions user : entities.getUserMentions()) {
+ //Map the twitter user object into an actor
+ Map<String, Object> actor = new HashMap<String, Object>();
+ actor.put("id", "id:twitter:" + user.getIdStr());
+ actor.put("displayName", user.getScreenName());
+
+ userMentions.add(actor);
+ }
+
+ extensions.put("user_mentions", userMentions);
+
+ List<LinkDetails> urls = new ArrayList<LinkDetails>();
+ for(Url url : entities.getUrls()) {
+ LinkDetails linkDetails = new LinkDetails();
+
+ linkDetails.setFinalURL(url.getExpandedUrl());
+ linkDetails.setNormalizedURL(url.getDisplayUrl());
+ linkDetails.setOriginalURL(url.getUrl());
+
+ urls.add(linkDetails);
+ }
+ extensions.put("urls", urls);
+
+ extensions.put("keywords", tweet.getText());
+ }
+
@Override
public List<Activity> deserializeAll(List<String> serializedList) {
return null;
}
- public static Actor buildActor(Tweet tweet) {
- Actor actor = new Actor();
- User user = tweet.getUser();
- actor.setId(formatId(
- Optional.fromNullable(
- user.getIdStr())
- .or(Optional.of(user.getId().toString()))
- .orNull()
- ));
- actor.setDisplayName(user.getScreenName());
- if (user.getUrl()!=null){
- actor.setUrl(user.getUrl());
- }
- return actor;
- }
-
public static List<String> getLinks(Tweet tweet) {
List<String> links = Lists.newArrayList();
if( tweet.getEntities().getUrls() != null ) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e29eadc9/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
index 6082fb8..769246e 100644
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
@@ -97,11 +97,22 @@
},
"hashtags": {
"type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
+ "items": {
+ "type": "object",
+ "javaType": "org.apache.streams.twitter.pojo.Hashtag",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "text": {
+ "type": "string"
+ },
+ "indices": {
+ "type": "array",
+ "items": [{
+ "type": "integer"
+ }]
+ }
+ }
+ }
},
"urls": {
"type": "array",
[7/8] git commit: STREAMS-93 | Moved duplicated activity/actor
methods to TwitterActivityUtil class. Ensured that those new methods were
comprehensive
Posted by mf...@apache.org.
STREAMS-93 | Moved duplicated activity/actor methods to TwitterActivityUtil class. Ensured that those new methods were comprehensive
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/3f59c3a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/3f59c3a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/3f59c3a3
Branch: refs/heads/master
Commit: 3f59c3a3a4e771b707546bafb22aa548171e2788
Parents: 44167ec
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Tue May 27 10:30:42 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Tue May 27 10:30:42 2014 -0500
----------------------------------------------------------------------
.../TwitterJsonActivitySerializer.java | 129 -------------------
.../TwitterJsonRetweetActivitySerializer.java | 83 +-----------
.../TwitterJsonTweetActivitySerializer.java | 80 +-----------
.../serializer/util/TwitterActivityUtil.java | 77 ++++++++++-
4 files changed, 77 insertions(+), 292 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/3f59c3a3/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
index 8af6023..cae2ff8 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
@@ -1,49 +1,15 @@
package org.apache.streams.twitter.serializer;
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.AnnotationIntrospector;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector;
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.collect.Lists;
import org.apache.commons.lang.NotImplementedException;
import org.apache.streams.data.ActivitySerializer;
import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.jackson.StreamsJacksonModule;
import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Provider;
-import org.apache.streams.twitter.Url;
import org.apache.streams.twitter.pojo.*;
import org.apache.streams.twitter.provider.TwitterEventClassifier;
-import org.apache.streams.urls.LinkDetails;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.io.Serializable;
-import java.util.Map;
-import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
-
-/**
- * Created by sblackmon on 3/26/14.
- */
public class TwitterJsonActivitySerializer implements ActivitySerializer<String>, Serializable
{
@@ -86,99 +52,4 @@ public class TwitterJsonActivitySerializer implements ActivitySerializer<String>
public List<Activity> deserializeAll(List<String> serializedList) {
throw new NotImplementedException();
}
-
- public static Actor buildActorRetweet(Retweet tweet) {
- return buildActor(tweet.getUser());
- }
-
- public static Actor buildActorTweet(Tweet tweet) {
- return buildActor(tweet.getUser());
- }
-
- public static Actor buildActor(User user) {
- Actor actor = new Actor();
-
- actor.setId(formatId(
- Optional.fromNullable(
- user.getIdStr())
- .or(Optional.of(user.getId().toString()))
- .orNull()
- ));
- actor.setDisplayName(user.getScreenName());
- if (user.getUrl()!=null){
- actor.setUrl(user.getUrl());
- }
-
- actor.setDisplayName(user.getName());
- actor.setSummary(user.getDescription());
-
- Map<String, Object> extensions = new HashMap<String, Object>();
- extensions.put("location", user.getLocation());
- extensions.put("posts", user.getStatusesCount());
- extensions.put("favorites", user.getFavouritesCount());
- extensions.put("followers", user.getFollowersCount());
-
- Map<String, Object> image = new HashMap<String, Object>();
- image.put("url", user.getProfileImageUrlHttps());
-
- extensions.put("image", image);
- extensions.put("screenName", user.getScreenName());
-
- actor.setAdditionalProperty("extensions", extensions);
-
- return actor;
- }
-
- public static void addTwitterExtensions(Activity activity, Tweet tweet) {
- Map<String, Object> extensions = ensureExtensions(activity);
-
- List<String> hashtags = new ArrayList<String>();
- for(Hashtag hashtag : tweet.getEntities().getHashtags()) {
- hashtags.add(hashtag.getText());
- }
- extensions.put("hashtags", hashtags);
-
- Map<String, Object> likes = new HashMap<String, Object>();
- likes.put("perspectival", tweet.getFavorited());
- likes.put("count", tweet.getAdditionalProperties().get("favorite_count"));
-
- extensions.put("likes", likes);
-
- Map<String, Object> rebroadcasts = new HashMap<String, Object>();
- rebroadcasts.put("perspectival", tweet.getRetweeted());
- rebroadcasts.put("count", tweet.getRetweetCount());
-
- extensions.put("rebroadcasts", rebroadcasts);
-
- List<Map<String, Object>> userMentions = new ArrayList<Map<String, Object>>();
- Entities entities = tweet.getEntities();
-
- for(UserMentions user : entities.getUserMentions()) {
- //Map the twitter user object into an actor
- Map<String, Object> actor = new HashMap<String, Object>();
- actor.put("id", "id:twitter:" + user.getIdStr());
- actor.put("displayName", user.getScreenName());
-
- userMentions.add(actor);
- }
-
- extensions.put("user_mentions", userMentions);
-
- extensions.put("keywords", tweet.getText());
- }
-
- public static Provider getProvider() {
- Provider provider = new Provider();
- provider.setId("id:providers:twitter");
- return provider;
- }
-
- public static void addTwitterExtension(Activity activity, ObjectNode event) {
- Map<String, Object> extensions = org.apache.streams.data.util.ActivityUtil.ensureExtensions(activity);
- extensions.put("twitter", event);
- }
-
- public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/3f59c3a3/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
index a86c252..a286213 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
@@ -2,46 +2,23 @@ package org.apache.streams.twitter.serializer;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Optional;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
import org.apache.streams.data.ActivitySerializer;
import org.apache.streams.exceptions.ActivitySerializerException;
import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.twitter.Url;
-import org.apache.streams.twitter.pojo.*;
-import org.apache.streams.urls.LinkDetails;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.streams.twitter.pojo.Retweet;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.io.Serializable;
import java.util.List;
-import java.util.Map;
-import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerializer.*;
-import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
+import static org.apache.streams.twitter.serializer.util.TwitterActivityUtil.*;
-/**
- * Created with IntelliJ IDEA.
- * User: mdelaet
- * Date: 9/30/13
- * Time: 9:24 AM
- * To change this template use File | Settings | File Templates.
- */
public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<String>, Serializable {
public TwitterJsonRetweetActivitySerializer() {
}
- private final static Logger LOGGER = LoggerFactory.getLogger(TwitterJsonRetweetActivitySerializer.class);
-
@Override
public String serializationFormat() {
return null;
@@ -66,40 +43,7 @@ public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<
}
Activity activity = new Activity();
- activity.setActor(buildActorRetweet(retweet));
- activity.setVerb("share");
- Tweet retweetStatus = retweet.getRetweetedStatus();
-
- if( retweet.getRetweetedStatus() != null )
- activity.setObject(buildActivityObject(retweetStatus));
- activity.setId(TwitterJsonActivitySerializer.formatId(activity.getVerb(),
- Optional.fromNullable(
- retweet.getIdStr())
- .or(Optional.of(retweet.getId().toString()))
- .orNull()
- ));
- if(Strings.isNullOrEmpty(activity.getId()))
- throw new ActivitySerializerException("Unable to determine activity id");
- try {
- activity.setPublished(retweet.getCreatedAt());
- } catch( Exception e ) {
- throw new ActivitySerializerException("Unable to determine publishedDate", e);
- }
- //activity.setGenerator(buildGenerator(mapper));
- //activity.setIcon(getIcon(event));
- activity.setProvider(TwitterJsonActivitySerializer.getProvider());
- activity.setTitle("");
- try {
- activity.setContent(retweet.getRetweetedStatus().getText());
- } catch( Exception e ) {
- throw new ActivitySerializerException("Unable to determine content", e);
- }
- activity.setUrl("http://twitter.com/" + retweet.getUser().getIdStr() + "/status/" + retweet.getIdStr());
- activity.setLinks(TwitterJsonTweetActivitySerializer.getLinks(retweet.getRetweetedStatus()));
-
- addTwitterExtensions(activity, retweetStatus);
- addTwitterExtension(activity, mapper.convertValue(retweet, ObjectNode.class));
- addLocationExtension(activity, retweet);
+ updateActivity(retweet, activity);
return activity;
}
@@ -108,25 +52,4 @@ public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<
public List<Activity> deserializeAll(List<String> serializedList) {
return null;
}
-
- public static ActivityObject buildActivityObject(Tweet tweet) {
- ActivityObject actObj = new ActivityObject();
- String id = Optional.fromNullable(
- tweet.getIdStr())
- .or(Optional.of(tweet.getId().toString()))
- .orNull();
- if( id != null )
- actObj.setId(id);
- actObj.setObjectType("tweet");
- return actObj;
- }
-
- public static void addLocationExtension(Activity activity, Retweet retweet) {
- Map<String, Object> extensions = ensureExtensions(activity);
- Map<String, Object> location = new HashMap<String, Object>();
- location.put("id", TwitterJsonActivitySerializer.formatId(retweet.getIdStr()));
- location.put("coordinates", retweet.getCoordinates());
- extensions.put("location", location);
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/3f59c3a3/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
index 4e58694..31d0e96 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
@@ -2,39 +2,18 @@ package org.apache.streams.twitter.serializer;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Optional;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
import org.apache.commons.lang.NotImplementedException;
import org.apache.streams.data.ActivitySerializer;
import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.twitter.Url;
-import org.apache.streams.twitter.pojo.*;
-import org.apache.streams.urls.LinkDetails;
-import twitter4j.HashtagEntity;
+import org.apache.streams.twitter.pojo.Tweet;
import java.io.IOException;
import java.io.Serializable;
-import java.util.*;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerializer.*;
-import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
+import static org.apache.streams.twitter.serializer.util.TwitterActivityUtil.*;
-/**
- * Created with IntelliJ IDEA.
- * User: mdelaet
- * Date: 9/30/13
- * Time: 9:24 AM
- * To change this template use File | Settings | File Templates.
- */
public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<String>, Serializable {
@Override
@@ -62,30 +41,7 @@ public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<St
Activity activity = new Activity();
- activity.setActor(buildActorTweet(tweet));
- activity.setVerb("post");
- activity.setId(formatId(activity.getVerb(),
- Optional.fromNullable(
- tweet.getIdStr())
- .or(Optional.of(tweet.getId().toString()))
- .orNull()));
- if(Strings.isNullOrEmpty(activity.getId()))
- throw new ActivitySerializerException("Unable to determine activity id");
- try {
- activity.setPublished(tweet.getCreatedAt());
- } catch( Exception e ) {
- throw new ActivitySerializerException("Unable to determine publishedDate", e);
- }
- activity.setTarget(buildTarget(tweet));
- activity.setProvider(getProvider());
- activity.setTitle("");
- activity.setContent(tweet.getText());
- activity.setUrl("http://twitter.com/" + tweet.getUser().getIdStr() + "/status/" + tweet.getIdStr());
- activity.setLinks(getLinks(tweet));
-
- addTwitterExtension(activity, mapper.convertValue(tweet, ObjectNode.class));
- addLocationExtension(activity, tweet);
- addTwitterExtensions(activity, tweet);
+ updateActivity(tweet, activity);
return activity;
}
@@ -94,34 +50,4 @@ public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<St
public List<Activity> deserializeAll(List<String> serializedList) {
return null;
}
-
- public static List<String> getLinks(Tweet tweet) {
- List<String> links = Lists.newArrayList();
- if( tweet.getEntities().getUrls() != null ) {
- for (Url url : tweet.getEntities().getUrls()) {
- links.add(url.getExpandedUrl());
- }
- }
- else
- System.out.println(" 0 links");
- return links;
- }
-
- public static ActivityObject buildTarget(Tweet tweet) {
- return null;
- }
-
- public static void addLocationExtension(Activity activity, Tweet tweet) {
- Map<String, Object> extensions = ensureExtensions(activity);
- Map<String, Object> location = new HashMap<String, Object>();
- location.put("id", formatId(
- Optional.fromNullable(
- tweet.getIdStr())
- .or(Optional.of(tweet.getId().toString()))
- .orNull()
- ));
- location.put("coordinates", tweet.getCoordinates());
- extensions.put("location", location);
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/3f59c3a3/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java
index 5651aea..9f4578f 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java
@@ -31,12 +31,10 @@ import org.apache.streams.pojo.json.ActivityObject;
import org.apache.streams.pojo.json.Actor;
import org.apache.streams.pojo.json.Provider;
import org.apache.streams.twitter.Url;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
+import org.apache.streams.twitter.pojo.*;
import org.apache.streams.twitter.serializer.StreamsTwitterMapper;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -78,11 +76,15 @@ public class TwitterActivityUtil {
}
activity.setTarget(buildTarget(tweet));
activity.setProvider(getProvider());
- activity.setUrl(String.format("http://twitter.com/%s/%s", tweet.getUser().getScreenName(),tweet.getIdStr()));
+ activity.setUrl(String.format("http://twitter.com/%s/%s/%s", tweet.getUser().getScreenName(),"/status/",tweet.getIdStr()));
+ activity.setTitle("");
+ activity.setContent(tweet.getText());
+ activity.setLinks(getLinks(tweet));
addTwitterExtension(activity, mapper.convertValue(tweet, ObjectNode.class));
addLocationExtension(activity, tweet);
+ addTwitterExtensions(activity, tweet);
}
/**
@@ -174,10 +176,28 @@ public class TwitterActivityUtil {
.or(Optional.of(user.getId().toString()))
.orNull()
));
- actor.setDisplayName(user.getScreenName());
+
+ actor.setDisplayName(user.getName());
+ actor.setAdditionalProperty("handle", user.getScreenName());
+ actor.setSummary(user.getDescription());
+
if (user.getUrl()!=null){
actor.setUrl(user.getUrl());
}
+
+ Map<String, Object> extensions = new HashMap<String, Object>();
+ extensions.put("location", user.getLocation());
+ extensions.put("posts", user.getStatusesCount());
+ extensions.put("favorites", user.getFavouritesCount());
+ extensions.put("followers", user.getFollowersCount());
+
+ Map<String, Object> image = new HashMap<String, Object>();
+ image.put("url", user.getProfileImageUrlHttps());
+
+ extensions.put("image", image);
+ extensions.put("screenName", user.getScreenName());
+
+ actor.setAdditionalProperty("extensions", extensions);
return actor;
}
@@ -252,4 +272,49 @@ public class TwitterActivityUtil {
public static String formatId(String... idparts) {
return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
}
+
+ /**
+ * Takes various parameters from the twitter object that are currently not part of teh
+ * activity schema and stores them in a generic extensions attribute
+ * @param activity
+ * @param tweet
+ */
+ public static void addTwitterExtensions(Activity activity, Tweet tweet) {
+ Map<String, Object> extensions = ensureExtensions(activity);
+
+ List<String> hashtags = new ArrayList<String>();
+ for(Hashtag hashtag : tweet.getEntities().getHashtags()) {
+ hashtags.add(hashtag.getText());
+ }
+ extensions.put("hashtags", hashtags);
+
+ Map<String, Object> likes = new HashMap<String, Object>();
+ likes.put("perspectival", tweet.getFavorited());
+ likes.put("count", tweet.getAdditionalProperties().get("favorite_count"));
+
+ extensions.put("likes", likes);
+
+ Map<String, Object> rebroadcasts = new HashMap<String, Object>();
+ rebroadcasts.put("perspectival", tweet.getRetweeted());
+ rebroadcasts.put("count", tweet.getRetweetCount());
+
+ extensions.put("rebroadcasts", rebroadcasts);
+
+ List<Map<String, Object>> userMentions = new ArrayList<Map<String, Object>>();
+ Entities entities = tweet.getEntities();
+
+ for(UserMentions user : entities.getUserMentions()) {
+ //Map the twitter user object into an actor
+ Map<String, Object> actor = new HashMap<String, Object>();
+ actor.put("id", "id:twitter:" + user.getIdStr());
+ actor.put("displayName", user.getName());
+ actor.put("handle", user.getScreenName());
+
+ userMentions.add(actor);
+ }
+
+ extensions.put("user_mentions", userMentions);
+
+ extensions.put("keywords", tweet.getText());
+ }
}
[8/8] git commit: STREAMS-93 | Properly setting image in actor object
Posted by mf...@apache.org.
STREAMS-93 | Properly setting image in actor object
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/0fc46069
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/0fc46069
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/0fc46069
Branch: refs/heads/master
Commit: 0fc460695251b91752f7f3ef0a803e83cefc6ab4
Parents: 3f59c3a
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Tue May 27 11:23:32 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Tue May 27 11:23:32 2014 -0500
----------------------------------------------------------------------
.../twitter/serializer/util/TwitterActivityUtil.java | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0fc46069/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java
index 9f4578f..8ead14c 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/util/TwitterActivityUtil.java
@@ -26,10 +26,7 @@ import com.google.common.base.Optional;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Provider;
+import org.apache.streams.pojo.json.*;
import org.apache.streams.twitter.Url;
import org.apache.streams.twitter.pojo.*;
import org.apache.streams.twitter.serializer.StreamsTwitterMapper;
@@ -191,10 +188,10 @@ public class TwitterActivityUtil {
extensions.put("favorites", user.getFavouritesCount());
extensions.put("followers", user.getFollowersCount());
- Map<String, Object> image = new HashMap<String, Object>();
- image.put("url", user.getProfileImageUrlHttps());
+ Image profileImage = new Image();
+ profileImage.setUrl(user.getProfileImageUrlHttps());
+ actor.setImage(profileImage);
- extensions.put("image", image);
extensions.put("screenName", user.getScreenName());
actor.setAdditionalProperty("extensions", extensions);
[4/8] git commit: STREAMS-93 | Removed dependency on
streams-processor-urls
Posted by mf...@apache.org.
STREAMS-93 | Removed dependency on streams-processor-urls
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/28fa93dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/28fa93dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/28fa93dc
Branch: refs/heads/master
Commit: 28fa93dcd3470aeb44134e0ccd19555f1ab86f87
Parents: 7637b60
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Thu May 22 16:18:29 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Thu May 22 16:18:29 2014 -0500
----------------------------------------------------------------------
streams-contrib/streams-provider-twitter/pom.xml | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/28fa93dc/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
index 4bd4240..c104810 100644
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@ -46,11 +46,6 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-processor-urls</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
<artifactId>streams-config</artifactId>
</dependency>
<dependency>
[3/8] git commit: STREAMS-93 | Removed url additions to extensions
attribute
Posted by mf...@apache.org.
STREAMS-93 | Removed url additions to extensions attribute
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/7637b60b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/7637b60b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/7637b60b
Branch: refs/heads/master
Commit: 7637b60b0b6930c6e8848e49e6b96944a748c033
Parents: 2446c98
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Thu May 22 15:51:18 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Thu May 22 15:51:18 2014 -0500
----------------------------------------------------------------------
.../serializer/TwitterJsonActivitySerializer.java | 12 ------------
1 file changed, 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7637b60b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
index 9685af5..8af6023 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
@@ -164,18 +164,6 @@ public class TwitterJsonActivitySerializer implements ActivitySerializer<String>
extensions.put("user_mentions", userMentions);
- List<LinkDetails> urls = new ArrayList<LinkDetails>();
- for(Url url : entities.getUrls()) {
- LinkDetails linkDetails = new LinkDetails();
-
- linkDetails.setFinalURL(url.getExpandedUrl());
- linkDetails.setNormalizedURL(url.getDisplayUrl());
- linkDetails.setOriginalURL(url.getUrl());
-
- urls.add(linkDetails);
- }
- extensions.put("urls", urls);
-
extensions.put("keywords", tweet.getText());
}
[6/8] git commit: Merge branch 'master' into STREAMS-93
Posted by mf...@apache.org.
Merge branch 'master' into STREAMS-93
Conflicts:
streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/44167ec7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/44167ec7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/44167ec7
Branch: refs/heads/master
Commit: 44167ec72c4b9024dc26f29a89f83849f6d6384b
Parents: 36099b6 4580d53
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Tue May 27 10:29:16 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Tue May 27 10:29:16 2014 -0500
----------------------------------------------------------------------
.../ElasticsearchPersistWriter.java | 45 +++-
.../apache/streams/json/JsonPathExtractor.java | 13 +-
.../json/test/JsonPathExtractorTwitterTest.java | 92 +++++++
.../src/test/resources/tweet.json | 1 +
.../provider/SysomosHeartbeatStream.java | 36 ++-
.../sysomos/provider/SysomosProvider.java | 83 +++++-
.../FetchAndReplaceTwitterProcessor.java | 173 +++++++++++++
.../provider/TwitterTimelineProvider.java | 53 +++-
.../serializer/StreamsTwitterMapper.java | 12 +
.../TwitterJsonDeleteActivitySerializer.java | 25 +-
...erJsonUserstreameventActivitySerializer.java | 7 +-
.../serializer/util/TwitterActivityUtil.java | 255 +++++++++++++++++++
.../twitter/test/TweetActivitySerDeTest.java | 1 -
streams-runtimes/streams-runtime-pig/pom.xml | 4 +-
.../streams/pig/StreamsComponentFactory.java | 7 +-
.../streams/pig/StreamsProcessDatumExec.java | 9 +-
.../streams/pig/StreamsProcessDocumentExec.java | 15 +-
.../streams/pig/StreamsSerializerExec.java | 8 +-
.../streams/pig/test/AppendStringProcessor.java | 59 +++++
.../streams/pig/test/CopyThriceProcessor.java | 7 +-
.../streams/pig/test/DoNothingProcessor.java | 9 +-
.../pig/test/PigProcessDocumentTest.java | 27 ++
.../resources/pigprocessdocumentappendtest.pig | 4 +
.../test/resources/pigprocessdocumenttest.pig | 4 +-
24 files changed, 852 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
[5/8] git commit: Merge branch 'master' into STREAMS-93
Posted by mf...@apache.org.
Merge branch 'master' into STREAMS-93
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/36099b6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/36099b6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/36099b6b
Branch: refs/heads/master
Commit: 36099b6b4d4fe75353a0a6a9b5d485635968d379
Parents: 28fa93d a2182b1
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Thu May 22 17:43:26 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Thu May 22 17:43:26 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 1 +
.../ElasticsearchPersistWriter.java | 121 +++++++++----
.../streams/hdfs/WebHdfsPersistReaderTask.java | 3 +-
streams-contrib/streams-processor-json/pom.xml | 86 ++++++++++
.../apache/streams/json/JsonPathExtractor.java | 140 +++++++++++++++
.../org/apache/streams/json/JsonPathFilter.java | 171 +++++++++++++++++++
.../json/test/JsonPathExtractorTest.java | 101 +++++++++++
.../src/test/resources/books.json | 21 +++
.../src/main/jsonschema/com/twitter/tweet.json | 21 +--
.../org/apache/streams/core/StreamsDatum.java | 4 +
.../local/builders/LocalStreamBuilder.java | 3 +-
.../local/tasks/StreamsProviderTask.java | 5 +-
.../apache/streams/local/tasks/StreamsTask.java | 10 +-
.../local/builders/LocalStreamBuilderTest.java | 37 +++-
.../test/providers/EmptyResultSetProvider.java | 63 +++++++
streams-runtimes/streams-runtime-pig/pom.xml | 45 +++--
.../streams/pig/StreamsComponentFactory.java | 22 ++-
.../apache/streams/pig/StreamsPigBuilder.java | 24 ++-
.../streams/pig/StreamsProcessDatumExec.java | 127 ++++++++++++--
.../streams/pig/StreamsProcessDocumentExec.java | 69 ++++----
.../streams/pig/StreamsSerializerExec.java | 36 ++--
.../org/apache/streams/pig/StreamsStorage.java | 22 ++-
.../src/test/java/PigProcessorTest.java | 32 ----
.../src/test/java/PigSerializerTest.java | 40 -----
.../streams/pig/test/CopyThriceProcessor.java | 56 ++++++
.../streams/pig/test/DoNothingProcessor.java | 54 ++++++
.../streams/pig/test/PigProcessDatumTest.java | 99 +++++++++++
.../pig/test/PigProcessDocumentTest.java | 90 ++++++++++
.../streams/pig/test/PigSerializerTest.java | 62 +++++++
.../test/resources/pigprocessdatumcopytest.pig | 4 +
.../src/test/resources/pigprocessdatumtest.pig | 4 +
.../test/resources/pigprocessdocumenttest.pig | 4 +
.../src/test/resources/pigprocessortest.pig | 7 -
.../src/test/resources/pigserializertest.pig | 12 +-
34 files changed, 1384 insertions(+), 212 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/36099b6b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
----------------------------------------------------------------------
[2/8] git commit: STREAMS-93 | Moved duplicated addExtensions()
function up to TwitterJsonActivitySerializer class as a utility function
Posted by mf...@apache.org.
STREAMS-93 | Moved duplicated addExtensions() function up to TwitterJsonActivitySerializer class as a utility function
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/2446c98d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/2446c98d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/2446c98d
Branch: refs/heads/master
Commit: 2446c98def7eceb5c7b747bf73242ccea03ce280
Parents: e29eadc
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Thu May 22 12:17:27 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Thu May 22 12:17:27 2014 -0500
----------------------------------------------------------------------
.../TwitterJsonActivitySerializer.java | 50 ++++++++++++++++++++
.../TwitterJsonRetweetActivitySerializer.java | 50 --------------------
.../TwitterJsonTweetActivitySerializer.java | 50 --------------------
3 files changed, 50 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2446c98d/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
index 766aaa6..9685af5 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
@@ -129,6 +129,56 @@ public class TwitterJsonActivitySerializer implements ActivitySerializer<String>
return actor;
}
+ public static void addTwitterExtensions(Activity activity, Tweet tweet) {
+ Map<String, Object> extensions = ensureExtensions(activity);
+
+ List<String> hashtags = new ArrayList<String>();
+ for(Hashtag hashtag : tweet.getEntities().getHashtags()) {
+ hashtags.add(hashtag.getText());
+ }
+ extensions.put("hashtags", hashtags);
+
+ Map<String, Object> likes = new HashMap<String, Object>();
+ likes.put("perspectival", tweet.getFavorited());
+ likes.put("count", tweet.getAdditionalProperties().get("favorite_count"));
+
+ extensions.put("likes", likes);
+
+ Map<String, Object> rebroadcasts = new HashMap<String, Object>();
+ rebroadcasts.put("perspectival", tweet.getRetweeted());
+ rebroadcasts.put("count", tweet.getRetweetCount());
+
+ extensions.put("rebroadcasts", rebroadcasts);
+
+ List<Map<String, Object>> userMentions = new ArrayList<Map<String, Object>>();
+ Entities entities = tweet.getEntities();
+
+ for(UserMentions user : entities.getUserMentions()) {
+ //Map the twitter user object into an actor
+ Map<String, Object> actor = new HashMap<String, Object>();
+ actor.put("id", "id:twitter:" + user.getIdStr());
+ actor.put("displayName", user.getScreenName());
+
+ userMentions.add(actor);
+ }
+
+ extensions.put("user_mentions", userMentions);
+
+ List<LinkDetails> urls = new ArrayList<LinkDetails>();
+ for(Url url : entities.getUrls()) {
+ LinkDetails linkDetails = new LinkDetails();
+
+ linkDetails.setFinalURL(url.getExpandedUrl());
+ linkDetails.setNormalizedURL(url.getDisplayUrl());
+ linkDetails.setOriginalURL(url.getUrl());
+
+ urls.add(linkDetails);
+ }
+ extensions.put("urls", urls);
+
+ extensions.put("keywords", tweet.getText());
+ }
+
public static Provider getProvider() {
Provider provider = new Provider();
provider.setId("id:providers:twitter");
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2446c98d/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
index da086a6..a86c252 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
@@ -104,56 +104,6 @@ public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<
return activity;
}
- public static void addTwitterExtensions(Activity activity, Tweet tweet) {
- Map<String, Object> extensions = ensureExtensions(activity);
-
- List<String> hashtags = new ArrayList<String>();
- for(Hashtag hashtag : tweet.getEntities().getHashtags()) {
- hashtags.add(hashtag.getText());
- }
- extensions.put("hashtags", hashtags);
-
- Map<String, Object> likes = new HashMap<String, Object>();
- likes.put("perspectival", tweet.getFavorited());
- likes.put("count", tweet.getAdditionalProperties().get("favorite_count"));
-
- extensions.put("likes", likes);
-
- Map<String, Object> rebroadcasts = new HashMap<String, Object>();
- rebroadcasts.put("perspectival", tweet.getRetweeted());
- rebroadcasts.put("count", tweet.getRetweetCount());
-
- extensions.put("rebroadcasts", rebroadcasts);
-
- List<Map<String, Object>> userMentions = new ArrayList<Map<String, Object>>();
- Entities entities = tweet.getEntities();
-
- for(UserMentions user : entities.getUserMentions()) {
- //Map the twitter user object into an actor
- Map<String, Object> actor = new HashMap<String, Object>();
- actor.put("id", "id:twitter:" + user.getIdStr());
- actor.put("displayName", user.getScreenName());
-
- userMentions.add(actor);
- }
-
- extensions.put("user_mentions", userMentions);
-
- List<LinkDetails> urls = new ArrayList<LinkDetails>();
- for(Url url : entities.getUrls()) {
- LinkDetails linkDetails = new LinkDetails();
-
- linkDetails.setFinalURL(url.getExpandedUrl());
- linkDetails.setNormalizedURL(url.getDisplayUrl());
- linkDetails.setOriginalURL(url.getUrl());
-
- urls.add(linkDetails);
- }
- extensions.put("urls", urls);
-
- extensions.put("keywords", tweet.getText());
- }
-
@Override
public List<Activity> deserializeAll(List<String> serializedList) {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2446c98d/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
index b8de327..4e58694 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
@@ -90,56 +90,6 @@ public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<St
return activity;
}
- public static void addTwitterExtensions(Activity activity, Tweet tweet) {
- Map<String, Object> extensions = ensureExtensions(activity);
-
- List<String> hashtags = new ArrayList<String>();
- for(Hashtag hashtag : tweet.getEntities().getHashtags()) {
- hashtags.add(hashtag.getText());
- }
- extensions.put("hashtags", hashtags);
-
- Map<String, Object> likes = new HashMap<String, Object>();
- likes.put("perspectival", tweet.getFavorited());
- likes.put("count", tweet.getAdditionalProperties().get("favorite_count"));
-
- extensions.put("likes", likes);
-
- Map<String, Object> rebroadcasts = new HashMap<String, Object>();
- rebroadcasts.put("perspectival", tweet.getRetweeted());
- rebroadcasts.put("count", tweet.getRetweetCount());
-
- extensions.put("rebroadcasts", rebroadcasts);
-
- List<Map<String, Object>> userMentions = new ArrayList<Map<String, Object>>();
- Entities entities = tweet.getEntities();
-
- for(UserMentions user : entities.getUserMentions()) {
- //Map the twitter user object into an actor
- Map<String, Object> actor = new HashMap<String, Object>();
- actor.put("id", "id:twitter:" + user.getIdStr());
- actor.put("displayName", user.getScreenName());
-
- userMentions.add(actor);
- }
-
- extensions.put("user_mentions", userMentions);
-
- List<LinkDetails> urls = new ArrayList<LinkDetails>();
- for(Url url : entities.getUrls()) {
- LinkDetails linkDetails = new LinkDetails();
-
- linkDetails.setFinalURL(url.getExpandedUrl());
- linkDetails.setNormalizedURL(url.getDisplayUrl());
- linkDetails.setOriginalURL(url.getUrl());
-
- urls.add(linkDetails);
- }
- extensions.put("urls", urls);
-
- extensions.put("keywords", tweet.getText());
- }
-
@Override
public List<Activity> deserializeAll(List<String> serializedList) {
return null;