You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/07/08 21:36:39 UTC
[1/2] git commit: Removed StreamsDatumCountable from procesor and
added twitter specific link extraction.
Repository: incubator-streams
Updated Branches:
refs/heads/master c50ce9175 -> d26ae9126
Removed StreamsDatumCountable from procesor and added twitter specific link extraction.
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/9d971ce0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/9d971ce0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/9d971ce0
Branch: refs/heads/master
Commit: 9d971ce0b79fb501af043edcc9cf9a314ec34dc0
Parents: c1246d9
Author: rebanks <re...@w2odigital.com>
Authored: Wed Jul 2 10:46:49 2014 -0500
Committer: rebanks <re...@w2odigital.com>
Committed: Wed Jul 2 10:46:49 2014 -0500
----------------------------------------------------------------------
.../DatasiftTypeConverterProcessor.java | 12 +-----
.../DatasiftTweetActivitySerializer.java | 42 +++++++++++++++++++-
2 files changed, 42 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9d971ce0/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessor.java
index e5b745f..203e5e8 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessor.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessor.java
@@ -34,7 +34,7 @@ import java.util.List;
/**
*
*/
-public class DatasiftTypeConverterProcessor implements StreamsProcessor,DatumStatusCountable {
+public class DatasiftTypeConverterProcessor implements StreamsProcessor {
private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftTypeConverterProcessor.class);
@@ -42,7 +42,6 @@ public class DatasiftTypeConverterProcessor implements StreamsProcessor,DatumSta
private Class outClass;
private DatasiftActivitySerializer datasiftInteractionActivitySerializer;
private DatasiftConverter converter;
- private DatumStatusCounter counter;
public final static String TERMINATE = new String("TERMINATE");
@@ -52,7 +51,6 @@ public class DatasiftTypeConverterProcessor implements StreamsProcessor,DatumSta
@Override
public List<StreamsDatum> process(StreamsDatum entry) {
- this.counter.incrementAttempt();
List<StreamsDatum> result = Lists.newLinkedList();
Object doc;
try {
@@ -62,9 +60,7 @@ public class DatasiftTypeConverterProcessor implements StreamsProcessor,DatumSta
}
} catch (Exception e) {
LOGGER.error("Exception converting Datasift Interaction to "+this.outClass.getName()+ " : {}", e);
- this.counter.incrementStatus(DatumStatus.FAIL);
}
- this.counter.incrementStatus(DatumStatus.SUCCESS);
return result;
}
@@ -80,7 +76,6 @@ public class DatasiftTypeConverterProcessor implements StreamsProcessor,DatumSta
LOGGER.warn("Using defaulting datasift converter");
this.converter = new DefaultConverter(this.outClass);
}
- this.counter = new DatumStatusCounter();
}
@Override
@@ -88,11 +83,6 @@ public class DatasiftTypeConverterProcessor implements StreamsProcessor,DatumSta
}
- @Override
- public DatumStatusCounter getDatumStatusCounter() {
- return this.counter;
- }
-
private class ActivityConverter implements DatasiftConverter {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9d971ce0/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
index 3406eb4..05b579b 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
@@ -70,7 +70,10 @@ public class DatasiftTweetActivitySerializer extends DatasiftDefaultActivitySeri
activity.setTitle(event.getInteraction().getTitle());
activity.setContent(event.getInteraction().getContent());
activity.setUrl(event.getInteraction().getLink());
- activity.setLinks(getLinks(event));
+ if(retweet)
+ activity.setLinks(getLinks(twitter.getRetweet()));
+ else
+ activity.setLinks(getLinks(twitter));
addDatasiftExtension(activity, event);
if( twitter.getGeo() != null) {
addLocationExtension(activity, twitter);
@@ -79,6 +82,43 @@ public class DatasiftTweetActivitySerializer extends DatasiftDefaultActivitySeri
return activity;
}
+ /**
+ * Get the links from this tweet as a list
+ * @param twitter
+ * @return the links from the tweet
+ */
+ public List<String> getLinks(Twitter twitter) {
+ return getLinks(twitter.getLinks());
+ }
+
+ /**
+ * Get the links from this tweet as a list
+ * @param retweet
+ * @return the links from the tweet
+ */
+ public List<String> getLinks(Retweet retweet) {
+ return getLinks(retweet.getLinks());
+ }
+
+ /**
+ * Converts the list of objects to a list of strings
+ * @param links
+ * @return
+ */
+ private List<String> getLinks(List<Object> links) {
+ if(links == null)
+ return Lists.newArrayList();
+ List<String> result = Lists.newLinkedList();
+ for(Object obj : links) {
+ if(obj instanceof String) {
+ result.add((String) obj);
+ } else {
+ LOGGER.warn("Links is not instance of String : {}", obj.getClass().getName());
+ }
+ }
+ return result;
+ }
+
public Actor buildActor(Datasift event, Twitter twitter) {
User user = twitter.getUser();
Actor actor = super.buildActor(event.getInteraction());
[2/2] git commit: Merge PR#48 'rbnks/STREAMS-123'
Posted by mf...@apache.org.
Merge PR#48 'rbnks/STREAMS-123'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d26ae912
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d26ae912
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d26ae912
Branch: refs/heads/master
Commit: d26ae91267ffa4228634408ae77b5a681b7298ec
Parents: c50ce91 9d971ce
Author: mfranklin <mf...@apache.org>
Authored: Tue Jul 8 15:17:05 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Tue Jul 8 15:17:05 2014 -0400
----------------------------------------------------------------------
.../DatasiftTypeConverterProcessor.java | 12 +-----
.../DatasiftTweetActivitySerializer.java | 42 +++++++++++++++++++-
2 files changed, 42 insertions(+), 12 deletions(-)
----------------------------------------------------------------------