You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/08/28 21:26:50 UTC
[05/15] git commit: channel serializers call super on default
serializer first hashtags working still problems with additionalProperties
channel serializers call super on default serializer first
hashtags working
still problems with additionalProperties
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d387b1d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d387b1d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d387b1d4
Branch: refs/heads/master
Commit: d387b1d42e41c54e95462a2c093ea9564a6df0b9
Parents: e8c85ee
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu Aug 7 17:05:43 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../streams-provider-datasift/pom.xml | 9 +-
.../serializer/DatasiftActivitySerializer.java | 5 +-
.../DatasiftDefaultActivitySerializer.java | 11 +-
.../DatasiftInstagramActivitySerializer.java | 124 +++++++++++++++++++
.../DatasiftTweetActivitySerializer.java | 6 +
.../main/jsonschema/com/datasift/Datasift.json | 96 +++++++++++---
.../DatasiftActivitySerializerTest.java | 4 +-
7 files changed, 232 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml
index 3b9f96f..b68e261 100644
--- a/streams-contrib/streams-provider-datasift/pom.xml
+++ b/streams-contrib/streams-provider-datasift/pom.xml
@@ -39,11 +39,18 @@
<groupId>org.apache.streams</groupId>
<artifactId>streams-provider-twitter</artifactId>
<version>${project.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-provider-instagram</artifactId>
+ <version>${project.version}</version>
+ <optional>true</optional>
</dependency>
<dependency>
<groupId>com.datasift.client</groupId>
<artifactId>datasift-java</artifactId>
- <version>3.0.0-Beta3.4</version>
+ <version>3.0.0-Beta3.5</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
index 7d644f0..1e4f63c 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
@@ -32,8 +32,9 @@ import java.util.List;
*/
public class DatasiftActivitySerializer implements ActivitySerializer<Datasift> {
- private static final DatasiftTweetActivitySerializer TWITTER_SERIALIZER = new DatasiftTweetActivitySerializer();
private static final DatasiftDefaultActivitySerializer DEFAULT_SERIALIZER = new DatasiftDefaultActivitySerializer();
+ private static final DatasiftTweetActivitySerializer TWITTER_SERIALIZER = new DatasiftTweetActivitySerializer();
+ private static final DatasiftInstagramActivitySerializer INSTAGRAM_SERIALIZER = new DatasiftInstagramActivitySerializer();
private static final ObjectMapper MAPPER = StreamsDatasiftMapper.getInstance();
@Override
@@ -50,6 +51,8 @@ public class DatasiftActivitySerializer implements ActivitySerializer<Datasift>
public Activity deserialize(Datasift serialized) throws ActivitySerializerException {
if(serialized.getTwitter() != null) {
return TWITTER_SERIALIZER.deserialize(serialized);
+ } else if(serialized.getInstagram() != null) {
+ return INSTAGRAM_SERIALIZER.deserialize(serialized);
} else {
return DEFAULT_SERIALIZER.deserialize(serialized);
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
index b70aa12..615ca34 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
@@ -109,7 +109,7 @@ public class DatasiftDefaultActivitySerializer implements ActivitySerializer<Dat
Activity activity = new Activity();
activity.setActor(buildActor(event.getInteraction()));
- activity.setVerb("post");
+ activity.setVerb(selectVerb(event));
activity.setObject(buildActivityObject(event.getInteraction()));
activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
activity.setTarget(buildTarget(event.getInteraction()));
@@ -128,6 +128,10 @@ public class DatasiftDefaultActivitySerializer implements ActivitySerializer<Dat
return activity;
}
+ private String selectVerb(Datasift event) {
+ return "post";
+ }
+
public Actor buildActor(Interaction interaction) {
Actor actor = new Actor();
org.apache.streams.datasift.interaction.Author author = interaction.getAuthor();
@@ -202,4 +206,9 @@ public class DatasiftDefaultActivitySerializer implements ActivitySerializer<Dat
extensions.put("location", location);
}
+ public static String firstStringIfNotNull(List<Object> list) {
+ if( list != null && list.size() > 0) {
+ return (String) list.get(0);
+ } else return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
new file mode 100644
index 0000000..54f14af
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
@@ -0,0 +1,124 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+package org.apache.streams.datasift.serializer;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.streams.data.util.RFC3339Utils;
+import org.apache.streams.datasift.Datasift;
+import org.apache.streams.datasift.instagram.From;
+import org.apache.streams.datasift.instagram.Instagram;
+import org.apache.streams.datasift.interaction.Interaction;
+import org.apache.streams.instagram.serializer.util.InstagramActivityUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.apache.streams.pojo.json.Image;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
+
+/**
+ *
+ */
+public class DatasiftInstagramActivitySerializer extends DatasiftDefaultActivitySerializer {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftInstagramActivitySerializer.class);
+
+ @Override
+ public Activity convert(Datasift event) {
+ Activity activity = super.convert(event);
+
+ Instagram instagram = event.getInstagram();
+
+ activity.setActor(buildActor(event, instagram));
+ activity.setObject(buildActivityObject(event.getInteraction()));
+ activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
+ activity.setTarget(buildTarget(event.getInteraction()));
+ activity.setPublished(event.getInteraction().getCreatedAt());
+ activity.setGenerator(buildGenerator(event.getInteraction()));
+ activity.setIcon(getIcon(event.getInteraction()));
+ activity.setProvider(InstagramActivityUtil.getProvider());
+ activity.setTitle(event.getInteraction().getTitle());
+ activity.setUrl(event.getInteraction().getLink());
+
+ activity.setVerb(selectVerb(event));
+ addInstagramExtensions(activity, instagram);
+ return activity;
+ }
+
+ /**
+ * Gets links from the object
+ * @return
+ */
+ private List<String> getLinks(Instagram instagram) {
+ List<String> result = Lists.newLinkedList();
+ result.add(instagram.getMedia().getImage());
+ result.add(instagram.getMedia().getVideo());
+ return result;
+ }
+
+ public Actor buildActor(Datasift event, Instagram instagram) {
+ Actor actor = super.buildActor(event.getInteraction());
+ From user = instagram.getFrom();
+
+ actor.setDisplayName(firstStringIfNotNull(user.getFullName()));
+ actor.setId(formatId(Optional.fromNullable(
+ user.getId())
+ .or(actor.getId())));
+
+ Image profileImage = new Image();
+ String profileUrl = null;
+ profileUrl = user.getProfilePicture();
+ profileImage.setUrl(profileUrl);
+ actor.setImage(profileImage);
+
+ return actor;
+ }
+
+ public void addInstagramExtensions(Activity activity, Instagram instagram) {
+ Map<String, Object> extensions = ensureExtensions(activity);
+ List<String> hashTags;
+ if(instagram.getMedia() != null) {
+ hashTags = instagram.getMedia().getTags();
+ extensions.put("hashtags", hashTags);
+ extensions.put("keywords", instagram.getMedia().getCaption());
+ }
+ }
+
+ private String selectVerb(Datasift event) {
+ if( event.getInteraction().getSubtype().equals("like"))
+ return "like";
+ else
+ return "post";
+ }
+
+ public static String formatId(String... idparts) {
+ return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
index b16aae2..3c7abda 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
@@ -19,6 +19,7 @@ under the License.
package org.apache.streams.datasift.serializer;
+import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -257,4 +258,9 @@ public class DatasiftTweetActivitySerializer extends DatasiftDefaultActivitySeri
return userMentions;
}
+
+ public static String formatId(String... idparts) {
+ return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
index 37a63a0..fdf1aac 100644
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
@@ -150,6 +150,77 @@
}
}
},
+ "instagram": {
+ "javaType": "org.apache.streams.datasift.instagram.Instagram",
+ "type": "object",
+ "properties": {
+ "from": {
+ "javaType": "org.apache.streams.datasift.instagram.From",
+ "type": "object",
+ "properties": {
+ "full_name": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "id": {
+ "type": "string"
+ },
+ "profile_picture": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "media": {
+ "javaType": "org.apache.streams.datasift.instagram.Media",
+ "type": "object",
+ "properties": {
+ "caption": {
+ "type": "string"
+ },
+ "created_time": {
+ "type": "number"
+ },
+ "filter": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "image": {
+ "type": "string"
+ },
+ "link": {
+ "type": "string"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "type": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ },
+ "video": {
+ "type": "string"
+ }
+ }
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ },
"interaction": {
"javaType": "org.apache.streams.datasift.interaction.Interaction",
"type": "object",
@@ -206,23 +277,10 @@
"link": {
"type": "string"
},
- "mention_ids": {
- "type": "array",
- "items": [
- {
- "type": "integer"
- }
- ]
- },
- "mentions": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
+ "received_at": {
+ "type": "long"
},
- "source": {
+ "subtype": {
"type": "string"
},
"schema": {
@@ -236,6 +294,9 @@
"source": {
"type": "string"
},
+ "subtype": {
+ "type": "string"
+ },
"tags": {
"type": "array",
"items": [
@@ -955,12 +1016,11 @@
},
"mention_ids": {
"type": "array",
- "items":[
+ "items": [
{
"type": "integer"
}
]
-
},
"mentions": {
"type": "array",
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
index daff2d7..baff556 100644
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
@@ -62,7 +62,6 @@ public class DatasiftActivitySerializerTest {
}
}
-
/**
* Test that the minimum number of things that an activity has
* @param json
@@ -70,9 +69,10 @@ public class DatasiftActivitySerializerTest {
private void testGeneralConversion(String json) throws Exception {
Activity activity = SERIALIZER.deserialize(json);
assertNotNull(json, activity.getId());
- assertTrue(json, activity.getId().contains("datasift"));
assertNotNull(json, activity.getPublished());
+ assertNotNull(json, activity.getProvider());
assertNotNull(json, activity.getUrl());
+ assertNotNull(json, activity.getVerb());
Actor actor = activity.getActor();
assertNotNull(json, actor);