You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/07/08 20:45:32 UTC
[1/9] git commit: Merge pull request #4 from apache/master
Repository: incubator-streams
Updated Branches:
refs/heads/instagram 815ce2abd -> 14f7050ba
Merge pull request #4 from apache/master
Merge Apache
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/2ee31575
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/2ee31575
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/2ee31575
Branch: refs/heads/instagram
Commit: 2ee31575eba481bf77c98a9edfe3d04eae333f16
Parents: 829f805 a33c215
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Mon Jun 23 14:03:00 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Mon Jun 23 14:03:00 2014 -0500
----------------------------------------------------------------------
.../org/apache/streams/s3/S3PersistReader.java | 9 +-
.../streams/console/ConsolePersistReader.java | 23 +
.../streams/console/ConsolePersistWriter.java | 18 +
.../elasticsearch/ElasticsearchClient.java | 20 +-
.../ElasticsearchClientManager.java | 18 +
.../ElasticsearchConfigurator.java | 18 +
.../ElasticsearchPersistReader.java | 26 +-
.../ElasticsearchPersistUpdater.java | 18 +
.../ElasticsearchPersistWriter.java | 1 +
.../ElasticsearchPersistWriterTask.java | 18 +
.../elasticsearch/ElasticsearchQuery.java | 20 +-
.../elasticsearch/PercolateProcessor.java | 20 +-
.../apache/streams/hbase/HbaseConfigurator.java | 18 +
.../streams/hbase/HbasePersistWriter.java | 18 +
.../streams/hbase/HbasePersistWriterTask.java | 18 +
.../apache/streams/hdfs/HdfsConfigurator.java | 18 +
.../streams/hdfs/WebHdfsPersistReader.java | 27 +-
.../streams/hdfs/WebHdfsPersistReaderTask.java | 18 +
.../streams/hdfs/WebHdfsPersistWriter.java | 18 +
.../streams/hdfs/WebHdfsPersistWriterTask.java | 18 +
.../apache/streams/kafka/KafkaConfigurator.java | 18 +
.../streams/kafka/KafkaPersistReader.java | 23 +
.../streams/kafka/KafkaPersistReaderTask.java | 18 +
.../streams/kafka/KafkaPersistWriter.java | 18 +
.../streams/kafka/KafkaPersistWriterTask.java | 18 +
.../streams/kafka/StreamsPartitioner.java | 18 +
.../apache/streams/mongo/MongoConfigurator.java | 18 +
.../streams/mongo/MongoPersistWriter.java | 18 +
.../apache/streams/json/JsonPathExtractor.java | 5 +-
.../org/apache/streams/urls/LinkResolver.java | 3 +-
.../urls/LinkResolverHelperFunctions.java | 1 +
.../streams/urls/LinkResolverProcessor.java | 3 +-
.../streams/urls/LinkHelperFunctionsTest.java | 18 +
.../streams/urls/TestLinkUnwinderProcessor.java | 18 +
.../streams-provider-datasift/pom.xml | 12 +-
.../streams/datasift/csdl/DatasiftCsdlUtil.java | 114 +
.../datasift/provider/DatasiftConverter.java | 1 +
.../provider/DatasiftStreamConfigurator.java | 2 +-
.../provider/DatasiftStreamProvider.java | 7 +-
.../DatasiftTypeConverterProcessor.java | 75 +-
.../serializer/DatasiftActivitySerializer.java | 193 +-
.../DatasiftDefaultActivitySerializer.java | 211 +
.../DatasiftTweetActivitySerializer.java | 234 +
.../main/jsonschema/com/datasift/Datasift.json | 44 +-
.../com/datasift/test/DatasiftSerDeTest.java | 37 +-
.../provider/DatasiftStreamProviderTest.java | 18 +
.../DatasiftTypeConverterProcessorTest.java | 20 +-
.../datasift/provider/ErrorHandlerTest.java | 18 +
.../datasift/provider/SubscriptionTest.java | 18 +
.../DatasiftActivitySerializerTest.java | 74 +
.../src/test/resources/amazon_datasift_json.txt | 10 +
.../src/test/resources/blog_datasift_json.txt | 719 +++
.../src/test/resources/board_datasift_json.txt | 4160 ++++++++++++++++++
.../test/resources/facebook_datasift_json.txt | 1843 ++++++++
.../src/test/resources/part-r-00000.json | 1724 ++++----
.../resources/rand_sample_datasift_json.txt | 1547 +++++++
.../resources/random_sample_datasift_json.txt | 1547 +++++++
.../src/test/resources/reddit_datasift_json.txt | 33 +
.../test/resources/twitter_datasift_json.txt | 1000 +++++
.../test/resources/wikipedia_datasift_json.txt | 252 ++
.../test/resources/youtube_datasift_json.txt | 7 +
...FacebookPublicFeedXmlActivitySerializer.java | 18 +
.../FacebookPostActivitySerializerTest.java | 1 +
.../facebook/test/FacebookPostSerDeTest.java | 18 +
.../facebook/test/FacebookEDCSerDeTest.java | 18 +
.../gnip/flickr/test/FlickrEDCSerDeTest.java | 18 +
.../com/gplus/api/GPlusActivitySerializer.java | 18 +
.../com/gplus/api/GPlusEDCAsActivityTest.java | 18 +
.../com/instagram/test/InstagramSerDeTest.java | 18 +
.../reddit/api/RedditActivitySerializer.java | 18 +
.../reddit/api/RedditEDCAsActivityJSONTest.java | 18 +
.../java/com/gnip/test/YouTubeEDCSerDeTest.java | 20 +-
.../com/gnip/test/YoutubeEDCAsActivityTest.java | 2 +-
.../ActivityXMLActivitySerializer.java | 18 +
.../PowerTrackActivitySerializer.java | 18 +
.../test/PowerTrackDeserializationTest.java | 18 +
.../com/google/gmail/GMailConfigurator.java | 18 +
.../gmail/provider/GMailImapProviderTask.java | 18 +
.../GMailMessageActivitySerializer.java | 18 +
.../google/gmail/provider/GMailProvider.java | 26 +-
.../gmail/provider/GMailRssProviderTask.java | 18 +
.../gplus/provider/GPlusActivitySerializer.java | 18 +
.../gplus/provider/GPlusConfigurator.java | 18 +
.../gplus/provider/GPlusEventProcessor.java | 18 +
.../provider/GPlusHistoryProviderTask.java | 18 +
.../google/gplus/provider/GPlusProvider.java | 23 +
.../gmail/test/GMailMessageSerDeTest.java | 18 +
.../data/MoreoverJsonActivitySerializer.java | 18 +
.../data/MoreoverXmlActivitySerializer.java | 18 +
.../streams/data/moreover/MoreoverClient.java | 18 +
.../data/moreover/MoreoverConfigurator.java | 18 +
.../streams/data/moreover/MoreoverProvider.java | 23 +
.../data/moreover/MoreoverProviderTask.java | 18 +
.../streams/data/moreover/MoreoverResult.java | 18 +
.../data/moreover/MoreoverResultSetWrapper.java | 18 +
.../apache/streams/data/util/MoreoverUtils.java | 18 +
.../MoreoverJsonActivitySerializerTest.java | 1 +
.../data/MoreoverXmlActivitySerializerTest.java | 18 +
.../rss/provider/RssEventClassifier.java | 18 +
.../streams/rss/provider/RssEventProcessor.java | 18 +
.../rss/provider/RssStreamConfigurator.java | 18 +
.../streams/rss/provider/RssStreamProvider.java | 24 +
.../rss/provider/RssStreamProviderTask.java | 18 +
.../serializer/SyndEntryActivitySerializer.java | 18 +
.../streams/rss/test/Top100FeedsTest.java | 18 +
.../sysomos/config/SysomosConfigurator.java | 39 +
.../sysomos/proessor/SysomosTypeConverter.java | 56 -
.../sysomos/processor/SysomosTypeConverter.java | 56 +
.../sysomos/provider/ContentRequestBuilder.java | 1 +
.../provider/SysomosHeartbeatStream.java | 130 +-
.../sysomos/provider/SysomosProvider.java | 50 +-
.../streams/sysomos/util/SysomosUtils.java | 2 +-
.../com/sysomos/test/SysomosJsonSerDeTest.java | 18 +
.../com/sysomos/test/SysomosXmlSerDeTest.java | 18 +
.../processor/TwitterEventProcessor.java | 18 +
.../processor/TwitterProfileProcessor.java | 31 +-
.../twitter/processor/TwitterTypeConverter.java | 18 +
.../twitter/provider/TwitterErrorHandler.java | 20 +-
.../provider/TwitterEventClassifier.java | 18 +
.../provider/TwitterStreamConfigurator.java | 18 +
.../twitter/provider/TwitterStreamProvider.java | 23 +
.../provider/TwitterStreamProviderTask.java | 18 +
.../provider/TwitterTimelineProvider.java | 32 +-
.../provider/TwitterTimelineProviderTask.java | 18 +
.../TwitterUserInformationProvider.java | 12 +-
.../serializer/StreamsTwitterMapper.java | 18 +
.../TwitterJsonActivitySerializer.java | 18 +
.../TwitterJsonDeleteActivitySerializer.java | 18 +
.../TwitterJsonRetweetActivitySerializer.java | 18 +
.../TwitterJsonTweetActivitySerializer.java | 18 +
...erJsonUserstreameventActivitySerializer.java | 18 +
.../streams/twitter/test/SimpleTweetTest.java | 18 +
.../twitter/test/TweetActivitySerDeTest.java | 18 +
.../streams/twitter/test/TweetSerDeTest.java | 18 +
.../test/TwitterEventClassifierTest.java | 18 +
.../twitter/test/TwitterStreamProviderTest.java | 18 +
.../org/apache/streams/core/DatumStatus.java | 18 +
.../streams/core/DatumStatusCountable.java | 18 +
.../apache/streams/core/DatumStatusCounter.java | 18 +
.../org/apache/streams/core/StreamBuilder.java | 18 +
.../org/apache/streams/core/StreamHandler.java | 18 +
.../org/apache/streams/core/StreamState.java | 18 +
.../apache/streams/core/StreamsOperation.java | 18 +
.../apache/streams/core/StreamsProvider.java | 38 +-
.../apache/streams/data/util/RFC3339Utils.java | 5 +
.../ActivityDeserializerException.java | 18 +
.../exceptions/ActivitySerializerException.java | 18 +
.../jackson/StreamsDateTimeDeserializer.java | 20 +-
.../jackson/StreamsDateTimeSerializer.java | 18 +
.../streams/jackson/StreamsJacksonMapper.java | 22 +-
.../streams/jackson/StreamsJacksonModule.java | 18 +
.../jackson/StreamsPeriodDeserializer.java | 18 +
.../jackson/StreamsPeriodSerializer.java | 20 +-
.../data/data/util/DateTimeSerDeTest.java | 18 +
.../local/builders/InvalidStreamException.java | 18 +
.../local/builders/LocalStreamBuilder.java | 18 +
.../streams/local/builders/StreamComponent.java | 18 +
.../streams/local/tasks/BaseStreamsTask.java | 18 +
.../tasks/LocalStreamProcessMonitorThread.java | 18 +
.../tasks/StatusCounterMonitorRunnable.java | 18 +
.../local/tasks/StatusCounterMonitorThread.java | 18 +
.../streams/local/tasks/StreamsMergeTask.java | 18 +
.../local/tasks/StreamsPersistWriterTask.java | 18 +
.../local/tasks/StreamsProcessorTask.java | 18 +
.../local/tasks/StreamsProviderTask.java | 18 +
.../apache/streams/local/tasks/StreamsTask.java | 18 +
.../local/builders/LocalStreamBuilderTest.java | 18 +
.../local/builders/ToyLocalBuilderExample.java | 18 +
.../streams/local/tasks/BasicTasksTest.java | 18 +
.../test/processors/DoNothingProcessor.java | 18 +
.../PassthroughDatumCounterProcessor.java | 18 +
.../test/providers/EmptyResultSetProvider.java | 5 +
.../test/providers/NumericMessageProvider.java | 23 +
.../local/test/writer/DatumCounterWriter.java | 18 +
.../local/test/writer/DoNothingWriter.java | 18 +
.../local/test/writer/SystemOutWriter.java | 18 +
.../component/ExpectedDatumsPersistWriter.java | 18 +
.../test/component/FileReaderProvider.java | 23 +
.../test/component/StreamsDatumConverter.java | 18 +
.../component/StringToDocumentConverter.java | 18 +
.../tests/TestComponentsLocalStream.java | 18 +
.../tests/TestExpectedDatumsPersitWriter.java | 18 +
.../component/tests/TestFileReaderProvider.java | 18 +
.../trident/StreamsPersistWriterState.java | 18 +
.../storm/trident/StreamsProcessorFunction.java | 18 +
.../storm/trident/StreamsProviderSpout.java | 20 +-
.../org/apache/streams/util/ComponentUtils.java | 18 +
.../java/org/apache/streams/util/DateUtil.java | 18 +
.../apache/streams/util/SerializationUtil.java | 18 +
189 files changed, 15837 insertions(+), 1172 deletions(-)
----------------------------------------------------------------------
[6/9] git commit: STREAMS-105 | Updated the InstagramTypeConverter to
use the conversion utility functions provided in InstagramActivityUtil
Posted by mf...@apache.org.
STREAMS-105 | Updated the InstagramTypeConverter to use the conversion utility functions provided in InstagramActivityUtil
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/fa3f9220
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/fa3f9220
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/fa3f9220
Branch: refs/heads/instagram
Commit: fa3f92200d19605e931954a154e59123d1a36f03
Parents: 1163653
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Wed Jul 2 10:48:35 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Wed Jul 2 10:48:35 2014 -0500
----------------------------------------------------------------------
.../processor/InstagramTypeConverter.java | 116 +++----------------
.../serializer/util/InstagramActivityUtil.java | 32 +++--
2 files changed, 34 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/fa3f9220/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/processor/InstagramTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/processor/InstagramTypeConverter.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/processor/InstagramTypeConverter.java
index 14260e3..7fb1ec6 100644
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/processor/InstagramTypeConverter.java
+++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/processor/InstagramTypeConverter.java
@@ -18,52 +18,34 @@
package org.apache.streams.instagram.processor;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.instagram.serializer.InstagramJsonActivitySerializer;
-import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.instagram.serializer.util.InstagramActivityUtil;
import org.apache.streams.pojo.json.Activity;
import org.jinstagram.entity.users.feed.MediaFeedData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.List;
import java.util.Queue;
-/**
- * Created by sblackmon on 12/10/13.
- */
public class InstagramTypeConverter implements StreamsProcessor {
public final static String STREAMS_ID = "InstagramTypeConverter";
private final static Logger LOGGER = LoggerFactory.getLogger(InstagramTypeConverter.class);
- private ObjectMapper mapper;
-
private Queue<MediaFeedData> inQueue;
private Queue<StreamsDatum> outQueue;
- private Class inClass;
- private Class outClass;
-
- private InstagramJsonActivitySerializer instagramJsonActivitySerializer;
+ private InstagramActivityUtil instagramActivityUtil;
private int count = 0;
public final static String TERMINATE = new String("TERMINATE");
- public InstagramTypeConverter(Class inClass, Class outClass) {
- this.inClass = inClass;
- this.outClass = outClass;
+ public InstagramTypeConverter() {
}
public Queue<StreamsDatum> getProcessorOutputQueue() {
@@ -74,100 +56,31 @@ public class InstagramTypeConverter implements StreamsProcessor {
inQueue = inputQueue;
}
- public Object convert(ObjectNode event, Class inClass, Class outClass) throws ActivitySerializerException, JsonProcessingException {
-
- Object result = null;
-
- if( outClass.equals( Activity.class )) {
- LOGGER.debug("ACTIVITY");
- result = instagramJsonActivitySerializer.deserialize(
- mapper.writeValueAsString(event));
- } else if( outClass.equals( ObjectNode.class )) {
- LOGGER.debug("OBJECTNODE");
- result = mapper.convertValue(event, ObjectNode.class);
- } else if( outClass.equals( String.class )) {
- LOGGER.debug("OBJECTNODE");
- result = mapper.writeValueAsString(event);
- }
-
-
- // no supported conversion were applied
- if( result != null ) {
- count ++;
- return result;
- }
-
- LOGGER.debug("CONVERT FAILED");
-
- return null;
-
- }
-
- public boolean validate(Object document, Class klass) {
-
- // TODO
- return true;
- }
-
- public boolean isValidJSON(final String json) {
- boolean valid = false;
- try {
- final JsonParser parser = new ObjectMapper().getJsonFactory()
- .createJsonParser(json);
- while (parser.nextToken() != null) {
- }
- valid = true;
- } catch (JsonParseException jpe) {
- LOGGER.warn("validate: {}", jpe);
- } catch (IOException ioe) {
- LOGGER.warn("validate: {}", ioe);
- }
-
- return valid;
- }
-
@Override
public List<StreamsDatum> process(StreamsDatum entry) {
StreamsDatum result = null;
try {
-
Object item = entry.getDocument();
- ObjectNode node;
LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
- if( item instanceof String ) {
-
- // if the target is string, just pass-through
- if( String.class.equals(outClass)) {
- result = entry;
- }
- else {
- // first check for valid json
- node = (ObjectNode)mapper.readTree((String)item);
+ if(item instanceof MediaFeedData) {
+ //We don't need to use the mapper, since we have a process to convert between
+ //MediaFeedData objects and Activity objects already
+ Activity activity = new Activity();
- Object out = convert(node, String.class, outClass);
+ instagramActivityUtil.updateActivity((MediaFeedData)item, activity);
- if( out != null && validate(out, outClass))
- result = new StreamsDatum(out);
+ if(activity.getId() != null) {
+ result = new StreamsDatum(activity);
+ count++;
}
-
- } else if( item instanceof ObjectNode ) {
-
- // first check for valid json
- node = (ObjectNode)mapper.valueToTree(item);
-
- Object out = convert(node, ObjectNode.class, outClass);
-
- if( out != null && validate(out, outClass))
- result = new StreamsDatum(out);
-
}
-
} catch (Exception e) {
e.printStackTrace();
+ LOGGER.error("Exception while converting MediaFeedData to Activity: {}", e.getMessage());
}
if( result != null )
@@ -178,13 +91,12 @@ public class InstagramTypeConverter implements StreamsProcessor {
@Override
public void prepare(Object o) {
- mapper = new StreamsJacksonMapper();
- instagramJsonActivitySerializer = new InstagramJsonActivitySerializer();
+ instagramActivityUtil = new InstagramActivityUtil();
}
@Override
public void cleanUp() {
-
+ //noop
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/fa3f9220/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
index 0561ba7..bd926d5 100644
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
+++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
@@ -53,7 +53,9 @@ public class InstagramActivityUtil {
*/
public static void updateActivity(MediaFeedData item, Activity activity) throws ActivitySerializerException {
activity.setActor(buildActor(item));
- activity.setPublished(new DateTime(Long.parseLong(item.getCreatedTime()) * 1000));
+
+ if(item.getCreatedTime() != null)
+ activity.setPublished(new DateTime(Long.parseLong(item.getCreatedTime()) * 1000));
activity.setId(formatId(activity.getVerb(),
Optional.fromNullable(
@@ -78,16 +80,20 @@ public class InstagramActivityUtil {
public static Actor buildActor(MediaFeedData item) {
Actor actor = new Actor();
- Image image = new Image();
- image.setUrl(item.getUser().getProfilePictureUrl());
+ try {
+ Image image = new Image();
+ image.setUrl(item.getUser().getProfilePictureUrl());
- Map<String, Object> extensions = new HashMap<String, Object>();
- extensions.put("screenName", item.getUser().getUserName());
+ Map<String, Object> extensions = new HashMap<String, Object>();
+ extensions.put("screenName", item.getUser().getUserName());
- actor.setId(formatId(String.valueOf(item.getUser().getId())));
- actor.setImage(image);
- actor.setAdditionalProperty("extensions", extensions);
- actor.setAdditionalProperty("handle", item.getUser().getUserName());
+ actor.setId(formatId(String.valueOf(item.getUser().getId())));
+ actor.setImage(image);
+ actor.setAdditionalProperty("extensions", extensions);
+ actor.setAdditionalProperty("handle", item.getUser().getUserName());
+ } catch (Exception e) {
+ LOGGER.error("Exception trying to build actor object: {}", e.getMessage());
+ }
return actor;
}
@@ -244,9 +250,11 @@ public class InstagramActivityUtil {
addLocationExtension(activity, item);
- Map<String, Object> likes = new HashMap<String, Object>();
- likes.put("count", item.getLikes().getCount());
- extensions.put("likes", likes);
+ if(item.getLikes() != null) {
+ Map<String, Object> likes = new HashMap<String, Object>();
+ likes.put("count", item.getLikes().getCount());
+ extensions.put("likes", likes);
+ }
extensions.put("hashtags", item.getTags());
[9/9] git commit: STREAMS-122 | Responded to code review feedback
Posted by mf...@apache.org.
STREAMS-122 | Responded to code review feedback
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/14f7050b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/14f7050b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/14f7050b
Branch: refs/heads/instagram
Commit: 14f7050ba7de9bd783e6b2bf9c1dcf76da2d9eb2
Parents: 957e13f
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Mon Jul 7 11:08:17 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Mon Jul 7 11:08:17 2014 -0500
----------------------------------------------------------------------
.../serializer/util/InstagramActivityUtil.java | 45 ++++++++++++--------
1 file changed, 27 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/14f7050b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
index bd926d5..499d0e7 100644
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
+++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
@@ -24,10 +24,8 @@ import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import org.apache.streams.exceptions.ActivitySerializerException;
import org.apache.streams.pojo.json.*;
-import org.jinstagram.entity.common.ImageData;
-import org.jinstagram.entity.common.Images;
-import org.jinstagram.entity.common.VideoData;
-import org.jinstagram.entity.common.Videos;
+import org.jinstagram.entity.comments.CommentData;
+import org.jinstagram.entity.common.*;
import org.jinstagram.entity.users.feed.MediaFeedData;
import org.joda.time.DateTime;
import org.slf4j.Logger;
@@ -53,6 +51,7 @@ public class InstagramActivityUtil {
*/
public static void updateActivity(MediaFeedData item, Activity activity) throws ActivitySerializerException {
activity.setActor(buildActor(item));
+ activity.setVerb("post");
if(item.getCreatedTime() != null)
activity.setPublished(new DateTime(Long.parseLong(item.getCreatedTime()) * 1000));
@@ -109,6 +108,21 @@ public class InstagramActivityUtil {
actObj.setObjectType(item.getType());
actObj.setAttachments(buildActivityObjectAttachments(item));
+ Image standardResolution = new Image();
+ if(item.getType() == "image" && item.getImages() != null) {
+ ImageData standardResolutionData = item.getImages().getStandardResolution();
+ standardResolution.setHeight((double)standardResolutionData.getImageHeight());
+ standardResolution.setWidth((double)standardResolutionData.getImageWidth());
+ standardResolution.setUrl(standardResolutionData.getImageUrl());
+ } else if(item.getType() == "video" && item.getVideos() != null) {
+ VideoData standardResolutionData = item.getVideos().getStandardResolution();
+ standardResolution.setHeight((double)standardResolutionData.getHeight());
+ standardResolution.setWidth((double)standardResolutionData.getWidth());
+ standardResolution.setUrl(standardResolutionData.getUrl());
+ }
+
+ actObj.setImage(standardResolution);
+
return actObj;
}
@@ -212,11 +226,10 @@ public class InstagramActivityUtil {
if(item.getLocation() != null) {
Map<String, Object> coordinates = new HashMap<String, Object>();
coordinates.put("type", "Point");
- coordinates.put("coordinates", "[" + item.getLocation().getLatitude() + "," + item.getLocation().getLongitude() + "]");
+ coordinates.put("coordinates", "[" + item.getLocation().getLongitude() + "," + item.getLocation().getLatitude() + "]");
extensions.put("coordinates", coordinates);
}
-
}
/**
@@ -258,19 +271,15 @@ public class InstagramActivityUtil {
extensions.put("hashtags", item.getTags());
- Image standardResolution = new Image();
- if(item.getType() == "image" && item.getImages() != null) {
- ImageData standardResolutionData = item.getImages().getStandardResolution();
- standardResolution.setHeight((double)standardResolutionData.getImageHeight());
- standardResolution.setWidth((double)standardResolutionData.getImageWidth());
- standardResolution.setUrl(standardResolutionData.getImageUrl());
- } else if(item.getType() == "video" && item.getVideos() != null) {
- VideoData standardResolutionData = item.getVideos().getStandardResolution();
- standardResolution.setHeight((double)standardResolutionData.getHeight());
- standardResolution.setWidth((double)standardResolutionData.getWidth());
- standardResolution.setUrl(standardResolutionData.getUrl());
+ Comments comments = item.getComments();
+ String commentsConcat = "";
+ for(CommentData commentData : comments.getComments()) {
+ commentsConcat += " " + commentData.getText();
+ }
+ if(item.getCaption() != null) {
+ commentsConcat += " " + item.getCaption().getText();
}
- extensions.put("image", standardResolution);
+ extensions.put("keywords", commentsConcat);
}
}
\ No newline at end of file
[7/9] git commit: Merge branch 'instagram' into STREAMS-122
Posted by mf...@apache.org.
Merge branch 'instagram' into STREAMS-122
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/23bfcac8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/23bfcac8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/23bfcac8
Branch: refs/heads/instagram
Commit: 23bfcac82884d3280815f1168b82e263f38ef1fb
Parents: fa3f922 815ce2a
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Wed Jul 2 11:17:27 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Wed Jul 2 11:17:27 2014 -0500
----------------------------------------------------------------------
.../twitter/processor/TwitterTypeConverter.java | 2 +-
.../TwitterJsonActivitySerializer.java | 3 +
.../TwitterJsonUserActivitySerializer.java | 72 ++++++++++++++++++++
.../serializer/util/TwitterActivityUtil.java | 22 ++++++
4 files changed, 98 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[2/9] git commit: Merge pull request #5 from apache/master
Posted by mf...@apache.org.
Merge pull request #5 from apache/master
Merge From Apache
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d71b159d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d71b159d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d71b159d
Branch: refs/heads/instagram
Commit: d71b159d1996a939cccede70b509c1767478aa72
Parents: 2ee3157 34c95a6
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Fri Jun 27 16:43:57 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Fri Jun 27 16:43:57 2014 -0500
----------------------------------------------------------------------
pom.xml | 2 +-
streams-contrib/streams-provider-google/google-gmail/pom.xml | 2 +-
.../apache/streams/sysomos/provider/SysomosHeartbeatStream.java | 4 +++-
streams-contrib/streams-provider-twitter/pom.xml | 5 +++--
4 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[5/9] git commit: STREAMS-122 | Updated the InstagramActivityUtil
class to fully map Instagram MediaFeedData objects to Activities. Updated
tests so that this deserialization and mapping can be tested
Posted by mf...@apache.org.
STREAMS-122 | Updated the InstagramActivityUtil class to fully map Instagram MediaFeedData objects to Activities. Updated tests so that this deserialization and mapping can be tested
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/11636535
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/11636535
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/11636535
Branch: refs/heads/instagram
Commit: 116365355dad985da52092c85378b5bd7497e907
Parents: 7b301ce
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Tue Jul 1 17:12:57 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Tue Jul 1 17:12:57 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 1 +
.../InstagramJsonActivitySerializer.java | 22 ++-
.../serializer/util/InstagramActivityUtil.java | 170 ++++++++++++++++---
.../test/InstagramActivitySerDeTest.java | 24 ++-
.../src/test/resources/testMediaFeedObjects.txt | 2 +
5 files changed, 181 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/11636535/streams-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
index 620f68e..699274e 100644
--- a/streams-contrib/pom.xml
+++ b/streams-contrib/pom.xml
@@ -47,6 +47,7 @@
<module>streams-amazon-aws</module>
<!--<module>streams-processor-lucene</module>-->
<!--<module>streams-processor-tika</module>-->
+ <module>streams-provider-instagram</module>
<module>streams-processor-json</module>
<module>streams-processor-urls</module>
<module>streams-provider-datasift</module>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/11636535/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramJsonActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramJsonActivitySerializer.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramJsonActivitySerializer.java
index 8d92641..c5bbdf1 100644
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramJsonActivitySerializer.java
+++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramJsonActivitySerializer.java
@@ -18,14 +18,21 @@
package org.apache.streams.instagram.serializer;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang.NotImplementedException;
import org.apache.streams.data.ActivitySerializer;
import org.apache.streams.exceptions.ActivitySerializerException;
+import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.Activity;
+import org.jinstagram.entity.users.feed.MediaFeedData;
+import java.io.IOException;
import java.io.Serializable;
import java.util.List;
+import static org.apache.streams.instagram.serializer.util.InstagramActivityUtil.updateActivity;
+
public class InstagramJsonActivitySerializer implements ActivitySerializer<String>, Serializable
{
@@ -46,9 +53,20 @@ public class InstagramJsonActivitySerializer implements ActivitySerializer<Strin
@Override
public Activity deserialize(String serialized) throws ActivitySerializerException {
- Activity activity = null;
+ ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+ MediaFeedData mediaFeedData = null;
+
+ try {
+ mediaFeedData = mapper.readValue(serialized, MediaFeedData.class);
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ Activity activity = new Activity();
- // implement
+ updateActivity(mediaFeedData, activity);
return activity;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/11636535/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
index e71c43e..0561ba7 100644
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
+++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
@@ -19,16 +19,21 @@
package org.apache.streams.instagram.serializer.util;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Provider;
+import org.apache.streams.pojo.json.*;
+import org.jinstagram.entity.common.ImageData;
+import org.jinstagram.entity.common.Images;
+import org.jinstagram.entity.common.VideoData;
+import org.jinstagram.entity.common.Videos;
import org.jinstagram.entity.users.feed.MediaFeedData;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -39,7 +44,7 @@ import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
* Provides utilities for working with Activity objects within the context of Instagram
*/
public class InstagramActivityUtil {
-
+ private static final Logger LOGGER = LoggerFactory.getLogger(InstagramActivityUtil.class);
/**
* Updates the given Activity object with the values from the item
* @param item the object to use as the source
@@ -47,7 +52,22 @@ public class InstagramActivityUtil {
* @throws ActivitySerializerException
*/
public static void updateActivity(MediaFeedData item, Activity activity) throws ActivitySerializerException {
+ activity.setActor(buildActor(item));
+ activity.setPublished(new DateTime(Long.parseLong(item.getCreatedTime()) * 1000));
+
+ activity.setId(formatId(activity.getVerb(),
+ Optional.fromNullable(
+ item.getId())
+ .orNull()));
+
+ activity.setProvider(getProvider());
+ activity.setUrl(item.getLink());
+ activity.setObject(buildActivityObject(item));
+ if(item.getCaption() != null)
+ activity.setContent(item.getCaption().getText());
+
+ addInstagramExtensions(activity, item);
}
/**
@@ -57,6 +77,18 @@ public class InstagramActivityUtil {
*/
public static Actor buildActor(MediaFeedData item) {
Actor actor = new Actor();
+
+ Image image = new Image();
+ image.setUrl(item.getUser().getProfilePictureUrl());
+
+ Map<String, Object> extensions = new HashMap<String, Object>();
+ extensions.put("screenName", item.getUser().getUserName());
+
+ actor.setId(formatId(String.valueOf(item.getUser().getId())));
+ actor.setImage(image);
+ actor.setAdditionalProperty("extensions", extensions);
+ actor.setAdditionalProperty("handle", item.getUser().getUserName());
+
return actor;
}
@@ -67,18 +99,90 @@ public class InstagramActivityUtil {
*/
public static ActivityObject buildActivityObject(MediaFeedData item) {
ActivityObject actObj = new ActivityObject();
+
+ actObj.setObjectType(item.getType());
+ actObj.setAttachments(buildActivityObjectAttachments(item));
+
return actObj;
}
+ /**
+ * Builds all of the attachments associated with a MediaFeedData object
+ *
+ * @param item
+ * @return
+ */
+ public static List<ActivityObject> buildActivityObjectAttachments(MediaFeedData item) {
+ List<ActivityObject> attachments = new ArrayList<ActivityObject>();
+
+ addImageObjects(attachments, item);
+ addVideoObjects(attachments, item);
+
+ return attachments;
+ }
/**
- * Updates the content, and associated fields, with those from the given tweet
- * @param activity the target of the updates. Will receive all values from the tweet.
- * @param item the object to use as the source
- * @param verb the verb for the given activity's type
+ * Adds any image objects to the attachment field
+ * @param attachments
+ * @param item
+ */
+ public static void addImageObjects(List<ActivityObject> attachments, MediaFeedData item) {
+ Images images = item.getImages();
+
+ if(images != null) {
+ try {
+ ImageData thumbnail = images.getThumbnail();
+ ImageData lowResolution = images.getLowResolution();
+
+ ActivityObject thumbnailObject = new ActivityObject();
+ Image thumbnailImage = new Image();
+ thumbnailImage.setUrl(thumbnail.getImageUrl());
+ thumbnailImage.setHeight((double) thumbnail.getImageHeight());
+ thumbnailImage.setWidth((double) thumbnail.getImageWidth());
+ thumbnailObject.setImage(thumbnailImage);
+ thumbnailObject.setObjectType("image");
+
+ ActivityObject lowResolutionObject = new ActivityObject();
+ Image lowResolutionImage = new Image();
+ lowResolutionImage.setUrl(lowResolution.getImageUrl());
+ lowResolutionImage.setHeight((double) lowResolution.getImageHeight());
+ lowResolutionImage.setWidth((double) lowResolution.getImageWidth());
+ lowResolutionObject.setImage(lowResolutionImage);
+ lowResolutionObject.setObjectType("image");
+
+ attachments.add(thumbnailObject);
+ attachments.add(lowResolutionObject);
+ } catch (Exception e) {
+ LOGGER.error("Failed to add image objects: {}", e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Adds any video objects to the attachment field
+ * @param attachments
+ * @param item
*/
- public static void updateActivityContent(Activity activity, MediaFeedData item, String verb) {
+ public static void addVideoObjects(List<ActivityObject> attachments, MediaFeedData item) {
+ Videos videos = item.getVideos();
+
+ if(videos != null) {
+ try {
+ VideoData lowResolutionVideo = videos.getLowResolution();
+
+ ActivityObject lowResolutionVideoObject = new ActivityObject();
+ Image lowResolutionVideoImage = new Image();
+ lowResolutionVideoImage.setUrl(lowResolutionVideo.getUrl());
+ lowResolutionVideoImage.setHeight((double) lowResolutionVideo.getHeight());
+ lowResolutionVideoImage.setWidth((double) lowResolutionVideo.getWidth());
+ lowResolutionVideoObject.setImage(lowResolutionVideoImage);
+ lowResolutionVideoObject.setObjectType("video");
+ attachments.add(lowResolutionVideoObject);
+ } catch (Exception e) {
+ LOGGER.error("Failed to add video objects: {}", e.getMessage());
+ }
+ }
}
/**
@@ -98,7 +202,14 @@ public class InstagramActivityUtil {
*/
public static void addLocationExtension(Activity activity, MediaFeedData item) {
Map<String, Object> extensions = ensureExtensions(activity);
- Map<String, Object> location = new HashMap<String, Object>();
+
+ if(item.getLocation() != null) {
+ Map<String, Object> coordinates = new HashMap<String, Object>();
+ coordinates.put("type", "Point");
+ coordinates.put("coordinates", "[" + item.getLocation().getLatitude() + "," + item.getLocation().getLongitude() + "]");
+
+ extensions.put("coordinates", coordinates);
+ }
}
@@ -112,15 +223,7 @@ public class InstagramActivityUtil {
provider.setDisplayName("Instagram");
return provider;
}
- /**
- * Adds the given Instagram event to the activity as an extension
- * @param activity the Activity object to update
- * @param event the Instagram event to add as the extension
- */
- public static void addInstagramExtension(Activity activity, ObjectNode event) {
- Map<String, Object> extensions = org.apache.streams.data.util.ActivityUtil.ensureExtensions(activity);
- extensions.put("instagram", event);
- }
+
/**
* Formats the ID to conform with the Apache Streams activity ID convention
* @param idparts the parts of the ID to join
@@ -138,5 +241,28 @@ public class InstagramActivityUtil {
*/
public static void addInstagramExtensions(Activity activity, MediaFeedData item) {
Map<String, Object> extensions = ensureExtensions(activity);
+
+ addLocationExtension(activity, item);
+
+ Map<String, Object> likes = new HashMap<String, Object>();
+ likes.put("count", item.getLikes().getCount());
+ extensions.put("likes", likes);
+
+ extensions.put("hashtags", item.getTags());
+
+ Image standardResolution = new Image();
+ if(item.getType() == "image" && item.getImages() != null) {
+ ImageData standardResolutionData = item.getImages().getStandardResolution();
+ standardResolution.setHeight((double)standardResolutionData.getImageHeight());
+ standardResolution.setWidth((double)standardResolutionData.getImageWidth());
+ standardResolution.setUrl(standardResolutionData.getImageUrl());
+ } else if(item.getType() == "video" && item.getVideos() != null) {
+ VideoData standardResolutionData = item.getVideos().getStandardResolution();
+ standardResolution.setHeight((double)standardResolutionData.getHeight());
+ standardResolution.setWidth((double)standardResolutionData.getWidth());
+ standardResolution.setUrl(standardResolutionData.getUrl());
+ }
+
+ extensions.put("image", standardResolution);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/11636535/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/twitter/test/InstagramActivitySerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/twitter/test/InstagramActivitySerDeTest.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/twitter/test/InstagramActivitySerDeTest.java
index fcf5e81..075da80 100644
--- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/twitter/test/InstagramActivitySerDeTest.java
+++ b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/twitter/test/InstagramActivitySerDeTest.java
@@ -20,11 +20,12 @@ package org.apache.streams.twitter.test;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang.StringUtils;
+import org.apache.streams.instagram.serializer.util.InstagramDeserializer;
import org.apache.streams.instagram.serializer.InstagramJsonActivitySerializer;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.Activity;
+import org.jinstagram.entity.users.feed.MediaFeedData;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,6 +34,7 @@ import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
+import static org.apache.streams.instagram.serializer.util.InstagramActivityUtil.updateActivity;
import static org.hamcrest.CoreMatchers.*;
import static org.junit.Assert.assertThat;
@@ -46,16 +48,11 @@ import static org.junit.Assert.assertThat;
public class InstagramActivitySerDeTest {
private final static Logger LOGGER = LoggerFactory.getLogger(InstagramActivitySerDeTest.class);
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private InstagramJsonActivitySerializer instagramJsonActivitySerializer = new InstagramJsonActivitySerializer();
-
- // remove @Ignore after implementation
- @Ignore
@Test
- public void Tests()
- {
- InputStream is = InstagramActivitySerDeTest.class.getResourceAsStream("/test.txt");
+ public void Tests() {
+ InstagramDeserializer instagramDeserializer = new InstagramDeserializer("");
+ InputStream is = InstagramActivitySerDeTest.class.getResourceAsStream("/testMediaFeedObjects.txt");
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
@@ -66,13 +63,13 @@ public class InstagramActivitySerDeTest {
{
LOGGER.info("raw: {}", line);
- // convert to MediaFeedData?
- Activity activity = instagramJsonActivitySerializer.deserialize(line);
+ MediaFeedData mediaFeedData = instagramDeserializer.createObjectFromResponse(MediaFeedData.class, line);
- String activitystring = mapper.writeValueAsString(activity);
+ Activity activity = new Activity();
- LOGGER.info("activity: {}", activitystring);
+ LOGGER.info("activity: {}", activity.toString());
+ updateActivity(mediaFeedData, activity);
assertThat(activity, is(not(nullValue())));
assertThat(activity.getId(), is(not(nullValue())));
@@ -80,7 +77,6 @@ public class InstagramActivitySerDeTest {
assertThat(activity.getActor().getId(), is(not(nullValue())));
assertThat(activity.getVerb(), is(not(nullValue())));
assertThat(activity.getProvider(), is(not(nullValue())));
-
}
}
} catch( Exception e ) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/11636535/streams-contrib/streams-provider-instagram/src/test/resources/testMediaFeedObjects.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/test/resources/testMediaFeedObjects.txt b/streams-contrib/streams-provider-instagram/src/test/resources/testMediaFeedObjects.txt
index e69de29..b62e599 100644
--- a/streams-contrib/streams-provider-instagram/src/test/resources/testMediaFeedObjects.txt
+++ b/streams-contrib/streams-provider-instagram/src/test/resources/testMediaFeedObjects.txt
@@ -0,0 +1,2 @@
+{ "attribution":null, "tags":[ ], "type":"image", "location":null, "comments":{ "count":0, "data":[ ] }, "filter":"X-Pro II", "created_time":"1404162054", "link":"http://instagram.com/p/p4ez__syJi/", "likes":{ "count":0, "data":[ ] }, "images":{ "low_resolution":{ "url":"http://scontent-a.cdninstagram.com/hphotos-xpa1/t51.2885-15/10518155_283164271863608_1534480525_a.jpg", "width":306, "height":306 }, "thumbnail":{ "url":"http://scontent-a.cdninstagram.com/hphotos-xpa1/t51.2885-15/10518155_283164271863608_1534480525_s.jpg", "width":150, "height":150 }, "standard_resolution":{ "url":"http://scontent-a.cdninstagram.com/hphotos-xpa1/t51.2885-15/10518155_283164271863608_1534480525_n.jpg", "width":640, "height":640 } }, "users_in_photo":[ ], "caption":{ "created_time":"1404162054", "text":"Testing streams", "from":{ "username":"ktsafford", "profile_picture":"http://images.ak.instagram.com/profiles/anonymousUser.jpg", "id":"1412068271", "full_name":"ktsafford" }, "id":"754488452958068751"
}, "user_has_liked":false, "id":"754488452387644002_1412068271", "user":{ "username":"ktsafford", "website":"", "profile_picture":"http://images.ak.instagram.com/profiles/anonymousUser.jpg", "full_name":"ktsafford", "bio":"", "id":"1412068271" } }
+{ "type":"image", "users_in_photo":[ { "user":{ "username":"kevin", "full_name":"Kevin S", "id":"3", "profile_picture":"..." }, "position":{ "x":0.315, "y":0.9111 } } ], "filter":"Walden", "tags":[ ], "comments":{ "data":[ { "created_time":"1279332030", "text":"Love the sign here", "from":{ "username":"mikeyk", "full_name":"Mikey Krieger", "id":"4", "profile_picture":"http://distillery.s3.amazonaws.com/profiles/profile_1242695_75sq_1293915800.jpg" }, "id":"8" }, { "created_time":"1279341004", "text":"Chilako taco", "from":{ "username":"kevin", "full_name":"Kevin S", "id":"3", "profile_picture":"..." }, "id":"3" } ], "count":2 }, "caption":null, "likes":{ "count":1, "data":[ { "username":"mikeyk", "full_name":"Mikeyk", "id":"4", "profile_picture":"..." } ] }, "link":"http://instagr.am/p/D/", "user":{ "username":"kevin", "full_name":"Kevin S", "profile_picture":"...", "bio":"...", "website":"...", "id":"3" }, "created_time":"1279340983", "images":{ "low_resolution":{ "url":"http://dis
tillery.s3.amazonaws.com/media/2010/07/16/4de37e03aa4b4372843a7eb33fa41cad_6.jpg", "width":306, "height":306 }, "thumbnail":{ "url":"http://distillery.s3.amazonaws.com/media/2010/07/16/4de37e03aa4b4372843a7eb33fa41cad_5.jpg", "width":150, "height":150 }, "standard_resolution":{ "url":"http://distillery.s3.amazonaws.com/media/2010/07/16/4de37e03aa4b4372843a7eb33fa41cad_7.jpg", "width":612, "height":612 } }, "id":"3", "location":null }
[3/9] git commit: Merge pull request #6 from apache/master
Posted by mf...@apache.org.
Merge pull request #6 from apache/master
Merge Apache
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/3b98df84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/3b98df84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/3b98df84
Branch: refs/heads/instagram
Commit: 3b98df844ca87e79bf9cb88f16fbad361d9959ad
Parents: d71b159 ba96627
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Tue Jul 1 09:56:26 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Tue Jul 1 09:56:26 2014 -0500
----------------------------------------------------------------------
.../ElasticsearchConfigurator.java | 4 +
.../ElasticsearchPersistUpdater.java | 35 -
.../ElasticsearchPersistWriter.java | 705 +++++++------------
.../ElasticsearchPersistWriterTask.java | 56 --
.../ElasticsearchWriterConfiguration.json | 5 +
.../provider/SysomosHeartbeatStream.java | 2 +-
.../local/tasks/StreamsProviderTask.java | 51 +-
.../local/builders/LocalStreamBuilderTest.java | 52 +-
.../local/test/processors/SlowProcessor.java | 50 ++
.../test/providers/EmptyResultSetProvider.java | 2 +-
.../test/providers/NumericMessageProvider.java | 63 +-
.../test/component/FileReaderProvider.java | 52 +-
12 files changed, 419 insertions(+), 658 deletions(-)
----------------------------------------------------------------------
[8/9] git commit: STREAMS-122 | Added header
Posted by mf...@apache.org.
STREAMS-122 | Added header
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/957e13f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/957e13f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/957e13f9
Branch: refs/heads/instagram
Commit: 957e13f9d0bcf330d40e47691ce2c9b8a038781a
Parents: 23bfcac
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Wed Jul 2 11:19:57 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Wed Jul 2 11:19:57 2014 -0500
----------------------------------------------------------------------
.../serializer/util/InstagramDeserializer.java | 21 +++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/957e13f9/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramDeserializer.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramDeserializer.java
index b4aee2d..d4280bf 100644
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramDeserializer.java
+++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramDeserializer.java
@@ -1,11 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.instagram.serializer.util;
import org.jinstagram.Instagram;
import org.jinstagram.exceptions.InstagramException;
-/**
- * Created by rdouglas on 7/1/14.
- */
public class InstagramDeserializer extends Instagram{
public InstagramDeserializer(String test) {
super(test);
[4/9] git commit: Merge branch 'instagram' into STREAMS-122
Posted by mf...@apache.org.
Merge branch 'instagram' into STREAMS-122
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/7b301cea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/7b301cea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/7b301cea
Branch: refs/heads/instagram
Commit: 7b301ceaa358e6c572b95a0eff754a87cf9f6064
Parents: 3b98df8 a3f443f
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Tue Jul 1 17:11:28 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Tue Jul 1 17:11:28 2014 -0500
----------------------------------------------------------------------
.../streams-provider-instagram/README.md | 17 +
.../InstagramMediaDataActivitySerializer.mup | 894 +++++++++++++++++++
.../metadata/instagram_to_activity_mapping.png | Bin 0 -> 1123684 bytes
.../streams-provider-instagram/pom.xml | 139 +++
.../instagram/InstagramConfigurator.java | 78 ++
.../processor/InstagramTypeConverter.java | 190 ++++
.../provider/InstagramTimelineProvider.java | 409 +++++++++
.../InstagramJsonActivitySerializer.java | 60 ++
.../serializer/util/InstagramActivityUtil.java | 142 +++
.../serializer/util/InstagramDeserializer.java | 18 +
.../jsonschema/com/instagram/Instagram.json | 0
.../com/instagram/InstagramConfiguration.json | 20 +
.../InstagramUserInformationConfiguration.json | 17 +
.../src/main/resources/reference.conf | 5 +
.../test/InstagramActivitySerDeTest.java | 92 ++
.../src/test/resources/testMediaFeedObjects.txt | 0
.../src/test/resources/testtweets.txt | 695 ++++++++++++++
17 files changed, 2776 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7b301cea/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramDeserializer.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramDeserializer.java
index 0000000,0000000..b4aee2d
new file mode 100644
--- /dev/null
+++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramDeserializer.java
@@@ -1,0 -1,0 +1,18 @@@
++package org.apache.streams.instagram.serializer.util;
++
++import org.jinstagram.Instagram;
++import org.jinstagram.exceptions.InstagramException;
++
++/**
++ * Created by rdouglas on 7/1/14.
++ */
++public class InstagramDeserializer extends Instagram{
++ public InstagramDeserializer(String test) {
++ super(test);
++ }
++
++ @Override
++ public <T> T createObjectFromResponse(Class<T> clazz, String response) throws InstagramException {
++ return super.createObjectFromResponse(clazz, response);
++ }
++}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7b301cea/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/Instagram.json
----------------------------------------------------------------------
diff --cc streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/Instagram.json
index 0000000,0000000..e69de29
new file mode 100644
--- /dev/null
+++ b/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/Instagram.json
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7b301cea/streams-contrib/streams-provider-instagram/src/test/resources/testMediaFeedObjects.txt
----------------------------------------------------------------------
diff --cc streams-contrib/streams-provider-instagram/src/test/resources/testMediaFeedObjects.txt
index 0000000,0000000..e69de29
new file mode 100644
--- /dev/null
+++ b/streams-contrib/streams-provider-instagram/src/test/resources/testMediaFeedObjects.txt