You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/08/28 21:26:46 UTC
[01/15] git commit: additional fields streamlines instagram-specific
logic
Repository: incubator-streams
Updated Branches:
refs/heads/master 6b632def6 -> 67ab1ff6e
additional fields
streamlines instagram-specific logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/4cb20a16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/4cb20a16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/4cb20a16
Branch: refs/heads/master
Commit: 4cb20a161f0881fda4adfeeb18e5618cc0bf4c6e
Parents: be5a936
Author: sblackmon <sb...@w2odigital.com>
Authored: Mon Aug 11 12:34:29 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../DatasiftInstagramActivitySerializer.java | 22 +++----
.../main/jsonschema/com/datasift/Datasift.json | 63 ++++++++++++++++++++
2 files changed, 74 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4cb20a16/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
index 54f14af..4149fe3 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
@@ -56,15 +56,9 @@ public class DatasiftInstagramActivitySerializer extends DatasiftDefaultActivity
Instagram instagram = event.getInstagram();
activity.setActor(buildActor(event, instagram));
- activity.setObject(buildActivityObject(event.getInteraction()));
activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
- activity.setTarget(buildTarget(event.getInteraction()));
- activity.setPublished(event.getInteraction().getCreatedAt());
- activity.setGenerator(buildGenerator(event.getInteraction()));
- activity.setIcon(getIcon(event.getInteraction()));
activity.setProvider(InstagramActivityUtil.getProvider());
- activity.setTitle(event.getInteraction().getTitle());
- activity.setUrl(event.getInteraction().getLink());
+ activity.setLinks(getLinks(event.getInstagram()));
activity.setVerb(selectVerb(event));
addInstagramExtensions(activity, instagram);
@@ -77,8 +71,10 @@ public class DatasiftInstagramActivitySerializer extends DatasiftDefaultActivity
*/
private List<String> getLinks(Instagram instagram) {
List<String> result = Lists.newLinkedList();
- result.add(instagram.getMedia().getImage());
- result.add(instagram.getMedia().getVideo());
+ if( instagram.getMedia() != null ) {
+ result.add(instagram.getMedia().getImage());
+ result.add(instagram.getMedia().getVideo());
+ }
return result;
}
@@ -106,8 +102,12 @@ public class DatasiftInstagramActivitySerializer extends DatasiftDefaultActivity
if(instagram.getMedia() != null) {
hashTags = instagram.getMedia().getTags();
extensions.put("hashtags", hashTags);
- extensions.put("keywords", instagram.getMedia().getCaption());
+ extensions.put("keywords", activity.getContent());
+ } else {
+ extensions.put("keywords", activity.getContent());
+
}
+
}
private String selectVerb(Datasift event) {
@@ -118,7 +118,7 @@ public class DatasiftInstagramActivitySerializer extends DatasiftDefaultActivity
}
public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
+ return Joiner.on(":").join(Lists.asList("id:instagram", idparts));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4cb20a16/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
index fdf1aac..0c21302 100644
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
@@ -154,6 +154,46 @@
"javaType": "org.apache.streams.datasift.instagram.Instagram",
"type": "object",
"properties": {
+ "caption": {
+ "type": "object",
+ "properties": {
+ "created_time": {
+ "type": "number"
+ },
+ "id": {
+ "type": "string"
+ },
+ "text": {
+ "type": "string"
+ },
+ "from": {
+ "type": "object",
+ "properties": {
+ "full_name": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "id": {
+ "type": "string"
+ },
+ "profile_picture": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "created_at": {
+ "type": "string",
+ "format": "date-time"
+ },
"from": {
"javaType": "org.apache.streams.datasift.instagram.From",
"type": "object",
@@ -177,6 +217,23 @@
}
}
},
+ "id": {
+ "type": "string"
+ },
+ "images": {
+ "type": "object",
+ "properties": {
+ "low_resolution": {
+ "type": "object"
+ },
+ "standard_resolution": {
+ "type": "object"
+ },
+ "thumbnail": {
+ "type": "object"
+ }
+ }
+ },
"media": {
"javaType": "org.apache.streams.datasift.instagram.Media",
"type": "object",
@@ -216,6 +273,12 @@
}
}
},
+ "tags": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
"type": {
"type": "string"
}
[11/15] git commit: fixing imports and serialization tests after
schema breakup
Posted by sb...@apache.org.
fixing imports and serialization tests after schema breakup
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/4e3d769b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/4e3d769b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/4e3d769b
Branch: refs/heads/master
Commit: 4e3d769b3559fd116ac2d1f7c673b5325351fb50
Parents: 199881d
Author: sblackmon <sb...@apache.org>
Authored: Mon Aug 11 19:04:40 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:24 2014 -0500
----------------------------------------------------------------------
.../DatasiftDefaultActivitySerializer.java | 2 +-
.../datasift/twitter/DatasiftTwitter.json | 260 +------------------
.../datasift/twitter/DatasiftTwitterMedia.json | 126 +++++++++
.../DatasiftTypeConverterProcessorTest.java | 4 +-
.../DatasiftActivitySerializerTest.java | 3 +-
5 files changed, 136 insertions(+), 259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e3d769b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
index 615ca34..678f67b 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
@@ -6,7 +6,7 @@ import com.google.common.collect.Lists;
import org.apache.streams.data.ActivitySerializer;
import org.apache.streams.datasift.Datasift;
import org.apache.streams.datasift.interaction.Interaction;
-import org.apache.streams.datasift.interaction.Links;
+import org.apache.streams.datasift.links.Links;
import org.apache.streams.datasift.util.StreamsDatasiftMapper;
import org.apache.streams.pojo.json.*;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e3d769b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
index c671b2e..b11f81a 100644
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
@@ -95,135 +95,8 @@
"items" :
[
{
- "javaType" : "org.apache.streams.datasift.twitter.Media",
"type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "display_url" :
- {
- "type" : "string"
- },
- "expanded_url" :
- {
- "type" : "string"
- },
- "id" :
- {
- "type" : "integer"
- },
- "id_str" :
- {
- "type" : "string"
- },
- "media_url" :
- {
- "type" : "string"
- },
- "media_url_https" :
- {
- "type" : "string"
- },
- "sizes" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "large" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- },
- "medium" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- },
- "small" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- },
- "thumb" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- }
- }
- },
- "source_status_id" :
- {
- "type" : "integer"
- },
- "source_status_id_str" :
- {
- "type" : "string"
- },
- "type" :
- {
- "type" : "string"
- },
- "url" :
- {
- "type" : "string"
- }
- }
+ "$ref" : "DatasiftTwitterMedia.json"
}
]
},
@@ -350,133 +223,14 @@
},
"media" :
{
- "dynamic" : "true",
- "properties" :
- {
- "display_url" :
- {
- "type" : "string"
- },
- "expanded_url" :
- {
- "type" : "string"
- },
- "id" :
- {
- "type" : "integer"
- },
- "id_str" :
- {
- "type" : "string"
- },
- "media_url" :
- {
- "type" : "string"
- },
- "media_url_https" :
- {
- "type" : "string"
- },
- "sizes" :
- {
- "dynamic" : "true",
- "properties" :
+ "type" : "array",
+ "items" :
+ [
{
- "large" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- },
- "medium" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- },
- "small" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- },
- "thumb" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- }
+ "type" : "object",
+ "$ref" : "DatasiftTwitterMedia.json"
}
- },
- "source_status_id" :
- {
- "type" : "integer"
- },
- "source_status_id_str" :
- {
- "type" : "string"
- },
- "type" :
- {
- "type" : "string"
- },
- "url" :
- {
- "type" : "string"
- }
- }
+ ]
},
"mention_ids" :
{
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e3d769b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterMedia.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterMedia.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterMedia.json
new file mode 100644
index 0000000..2084932
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterMedia.json
@@ -0,0 +1,126 @@
+{
+ "type" : "object",
+ "javaType" : "org.apache.streams.datasift.twitter.Media",
+ "properties" :
+ {
+ "display_url" :
+ {
+ "type" : "string"
+ },
+ "expanded_url" :
+ {
+ "type" : "string"
+ },
+ "id" :
+ {
+ "type" : "integer"
+ },
+ "id_str" :
+ {
+ "type" : "string"
+ },
+ "media_url" :
+ {
+ "type" : "string"
+ },
+ "media_url_https" :
+ {
+ "type" : "string"
+ },
+ "sizes" :
+ {
+ "properties" :
+ {
+ "large" :
+ {
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ },
+ "medium" :
+ {
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ },
+ "small" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ },
+ "thumb" :
+ {
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ }
+ }
+ },
+ "source_status_id" :
+ {
+ "type" : "integer"
+ },
+ "source_status_id_str" :
+ {
+ "type" : "string"
+ },
+ "type" :
+ {
+ "type" : "string"
+ },
+ "url" :
+ {
+ "type" : "string"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e3d769b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java
index fac0f02..015f4e9 100644
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java
@@ -26,9 +26,7 @@ import org.junit.Test;
import java.util.List;
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.*;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e3d769b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
index baff556..5f9feed 100644
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
@@ -10,9 +10,8 @@ import org.junit.Test;
import java.util.Scanner;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
public class DatasiftActivitySerializerTest {
[06/15] git commit: user mention detection working v0 bean to help
configure managed sources via API
Posted by sb...@apache.org.
user mention detection working
v0 bean to help configure managed sources via API
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/6381dc97
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/6381dc97
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/6381dc97
Branch: refs/heads/master
Commit: 6381dc978b1a9794cd31dec909574015f820bd77
Parents: 9acc2f0
Author: sblackmon <sb...@w2odigital.com>
Authored: Fri Aug 8 11:47:55 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../regex/AbstractRegexExtensionExtractor.java | 13 ++-
.../com/datasift/DatasiftConfiguration.json | 115 ++++++++++++++++++-
2 files changed, 123 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6381dc97/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/AbstractRegexExtensionExtractor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/AbstractRegexExtensionExtractor.java b/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/AbstractRegexExtensionExtractor.java
index afd4972..ef31cdf 100644
--- a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/AbstractRegexExtensionExtractor.java
+++ b/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/AbstractRegexExtensionExtractor.java
@@ -19,11 +19,14 @@
package org.apache.streams.regex;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.Activity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,6 +49,7 @@ public abstract class AbstractRegexExtensionExtractor<T> implements StreamsProce
private final static Logger LOGGER = LoggerFactory.getLogger(AbstractRegexExtensionExtractor.class);
+ private final static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
private String pattern;
@@ -61,18 +65,23 @@ public abstract class AbstractRegexExtensionExtractor<T> implements StreamsProce
@Override
public List<StreamsDatum> process(StreamsDatum entry) {
- if (!(entry.getDocument() instanceof Activity)) {
+ Activity activity;
+ if (entry.getDocument() instanceof Activity) {
+ activity = (Activity) entry.getDocument();
+ } else if (entry.getDocument() instanceof ObjectNode) {
+ activity = mapper.convertValue(entry.getDocument(), Activity.class);
+ } else {
return Lists.newArrayList();
}
if (Strings.isNullOrEmpty(pattern)) {
prepare(null);
}
- Activity activity = (Activity) entry.getDocument();
Map<String, List<Integer>> matches = RegexUtils.extractMatches(pattern, activity.getContent());
Collection<T> entities = ensureTargetObject(activity);
for (String key : matches.keySet()) {
entities.add(prepareObject(key));
}
+ entry.setDocument(activity);
return Lists.newArrayList(entry);
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6381dc97/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
index f9a3574..ae9daa6 100644
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
@@ -2,8 +2,10 @@
"type": "object",
"$schema": "http://json-schema.org/draft-03/schema",
"id": "#",
- "javaType" : "org.apache.streams.datasift.DatasiftConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
+ "javaType": "org.apache.streams.datasift.DatasiftConfiguration",
+ "javaInterfaces": [
+ "java.io.Serializable"
+ ],
"properties": {
"apiKey": {
"type": "string"
@@ -13,10 +15,117 @@
},
"streamHash": {
"type": "array",
- "minItems": 1,
+ "minItems": 0,
"items": {
"type": "string"
}
+ },
+ "streamCsdl": {
+ "type": "array",
+ "minItems": 0,
+ "items": {
+ "type": "string"
+ }
+ },
+ "managedSources": {
+ "type": "array",
+ "minItems": 0,
+ "items": {
+ "javaType": "org.apache.streams.datasift.managed.ManagedSource",
+ "type": "object",
+ "properties": {
+ "auth": {
+ "type": "array",
+ "items": {
+ "javaType": "org.apache.streams.datasift.managed.ManagedSourceAuth",
+ "type": "object",
+ "properties": {
+ "identity_id": {
+ "type": "string"
+ },
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "value": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ },
+ "created_at": {
+ "type": "number"
+ },
+ "id": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "parameters": {
+ "javaType": "org.apache.streams.datasift.managed.ManagedSourceParameters",
+ "type": "object",
+ "properties": {
+ "comments": {
+ "type": "boolean"
+ },
+ "likes": {
+ "type": "boolean"
+ }
+ }
+ },
+ "resources": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "org.apache.streams.datasift.managed.ManagedSourceResource",
+ "type": "object",
+ "properties": {
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "distance": {
+ "type": "string"
+ },
+ "exact_match": {
+ "type": "string"
+ },
+ "foursq": {
+ "type": "string"
+ },
+ "lat": {
+ "type": "string"
+ },
+ "lng": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ },
+ "usermeta": {
+ "type": "string"
+ },
+ "value": {
+ "type": "number"
+ }
+ }
+ },
+ "resource_id": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "source_type": {
+ "type": "string"
+ },
+ "status": {
+ "type": "string"
+ }
+ }
+ }
}
}
}
\ No newline at end of file
[15/15] git commit: Merge remote-tracking branch 'apache/STREAMS-142'
Posted by sb...@apache.org.
Merge remote-tracking branch 'apache/STREAMS-142'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/67ab1ff6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/67ab1ff6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/67ab1ff6
Branch: refs/heads/master
Commit: 67ab1ff6ef6a1c5649c0cee164d397eeea9a8fb9
Parents: 6b632de 0536c98
Author: sblackmon <sb...@apache.org>
Authored: Thu Aug 28 14:14:45 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Thu Aug 28 14:14:45 2014 -0500
----------------------------------------------------------------------
pom.xml | 2 +-
streams-contrib/pom.xml | 1 +
.../streams-processor-jackson/pom.xml | 86 ++
.../CleanAdditionalPropertiesProcessor.java | 62 +
.../regex/AbstractRegexExtensionExtractor.java | 13 +-
.../streams-provider-datasift/pom.xml | 21 +-
.../streams/datasift/csdl/DatasiftCsdlUtil.java | 6 +-
.../DatasiftTypeConverterProcessor.java | 154 +++
.../provider/DatasiftManagedSourceSetup.java | 76 ++
.../provider/DatasiftStreamConfigurator.java | 15 +-
.../DatasiftTypeConverterProcessor.java | 167 ---
.../serializer/DatasiftActivitySerializer.java | 5 +-
.../DatasiftDefaultActivitySerializer.java | 13 +-
.../DatasiftInstagramActivitySerializer.java | 118 ++
.../DatasiftTweetActivitySerializer.java | 6 +
.../main/jsonschema/com/datasift/Datasift.json | 1103 ------------------
.../com/datasift/DatasiftConfiguration.json | 22 -
.../com/datasift/DatasiftPushConfiguration.json | 17 -
.../datasift/DatasiftStreamConfiguration.json | 17 -
.../com/datasift/DatasiftTwitterUser.json | 68 --
.../org/apache/streams/datasift/Datasift.json | 458 ++++++++
.../streams/datasift/DatasiftConfiguration.json | 133 +++
.../datasift/DatasiftPushConfiguration.json | 17 +
.../datasift/DatasiftStreamConfiguration.json | 17 +
.../datasift/facebook/DatasiftFacebook.json | 120 ++
.../datasift/instagram/DatasiftInstagram.json | 178 +++
.../interaction/DatasiftInteraction.json | 92 ++
.../datasift/twitter/DatasiftTwitter.json | 365 ++++++
.../datasift/twitter/DatasiftTwitterMedia.json | 126 ++
.../datasift/twitter/DatasiftTwitterUser.json | 68 ++
.../DatasiftTypeConverterProcessorTest.java | 5 +-
.../DatasiftActivitySerializerTest.java | 19 +-
.../test/resources/instagram_datasift_json.txt | 25 +
.../org/apache/streams/data/util/JsonUtil.java | 8 +-
.../jackson/StreamsDateTimeSerializer.java | 4 -
.../streams/jackson/StreamsJacksonMapper.java | 10 -
.../jackson/StreamsPeriodDeserializer.java | 1 -
.../jackson/StreamsPeriodSerializer.java | 2 -
.../data/data/util/DateTimeSerDeTest.java | 2 -
39 files changed, 2182 insertions(+), 1440 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/67ab1ff6/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/67ab1ff6/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
----------------------------------------------------------------------
[07/15] git commit: realized datasift managed source config objects
don't have setters, so creating streams beans that do.
Posted by sb...@apache.org.
realized datasift managed source config objects don't have setters, so creating streams beans that do.
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/0cf06a53
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/0cf06a53
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/0cf06a53
Branch: refs/heads/master
Commit: 0cf06a53519344eb0c03f458360c1523e2f87310
Parents: 1360223
Author: sblackmon <sb...@w2odigital.com>
Authored: Sun Aug 10 10:43:32 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../provider/DatasiftManagedSourceSetup.java | 17 +++-
.../com/datasift/DatasiftConfiguration.json | 96 +++++++++++++++++++-
2 files changed, 109 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0cf06a53/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
index 03bd7d1..9e2f99e 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
@@ -5,12 +5,15 @@ import com.datasift.client.FutureData;
import com.datasift.client.managedsource.ManagedSource;
import com.datasift.client.managedsource.ManagedSourceList;
import com.datasift.client.managedsource.sources.DataSource;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.streams.StreamsConfiguration;
import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.datasift.DatasiftConfiguration;
+import org.apache.streams.datasift.managed.StreamsManagedSource;
+import org.apache.streams.datasift.util.StreamsDatasiftMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,9 +31,11 @@ public class DatasiftManagedSourceSetup implements Runnable {
private static DatasiftConfiguration config = DatasiftStreamConfigurator.detectConfiguration(StreamsConfigurator.config);
+ private static final ObjectMapper MAPPER = StreamsDatasiftMapper.getInstance();
+
DataSiftClient client;
Map<String, ManagedSource> currentManagedSourceMap = Maps.newHashMap();
- List<ManagedSource> updatedManagedSourceList;
+ List<StreamsManagedSource> updatedManagedSourceList;
public static void main(String[] args) {
DatasiftManagedSourceSetup job = new DatasiftManagedSourceSetup();
@@ -46,9 +51,15 @@ public class DatasiftManagedSourceSetup implements Runnable {
updatedManagedSourceList = config.getManagedSources();
- for( ManagedSource source : updatedManagedSourceList ) {
+ for( StreamsManagedSource source : updatedManagedSourceList ) {
ManagedSource current = currentManagedSourceMap.get( source.getId() );
- ManagedSource updated = client.managedSource().update(current.getName(), (DataSource) source, current).sync();
+ LOGGER.info( "CURRENT: " + current );
+ // merge 'em
+ ManagedSource working = MAPPER.convertValue(source, ManagedSource.class);
+ LOGGER.info( "WORKING: " + working );
+ ManagedSource updated = client.managedSource().update(current.getName(), (DataSource) working, current).sync();
+ LOGGER.info( "UPDATED: " + updated );
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0cf06a53/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
index e024f43..9a0f027 100644
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
@@ -31,8 +31,102 @@
"type": "array",
"minItems": 0,
"items": {
+ "javaType": "org.apache.streams.datasift.managed.StreamsManagedSource",
"type": "object",
- "javaType": "com.datasift.client.managedsource.ManagedSource"
+ "properties": {
+ "auth": {
+ "type": "array",
+ "items": {
+ "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceAuth",
+ "type": "object",
+ "properties": {
+ "identity_id": {
+ "type": "string"
+ },
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "value": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ },
+ "created_at": {
+ "type": "number"
+ },
+ "id": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "parameters": {
+ "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceParams",
+ "type": "object",
+ "properties": {
+ "comments": {
+ "type": "boolean"
+ },
+ "likes": {
+ "type": "boolean"
+ }
+ }
+ },
+ "resources": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceResource",
+ "type": "object",
+ "properties": {
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "distance": {
+ "type": "number"
+ },
+ "exact_match": {
+ "type": "string"
+ },
+ "foursq": {
+ "type": "string"
+ },
+ "lat": {
+ "type": "number"
+ },
+ "lng": {
+ "type": "number"
+ },
+ "type": {
+ "type": "string",
+ "enum" : ["user", "tag", "area", "location", "popular"]
+ },
+ "usermeta": {
+ "type": "string"
+ },
+ "value": {
+ "type": "number"
+ }
+ }
+ },
+ "resource_id": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "source_type": {
+ "type": "string",
+ "enum" : ["facebook_page", "googleplus", "instagram", "yammer"]
+ },
+ "status": {
+ "type": "string"
+ }
+ }
}
}
}
[10/15] git commit: utilize improved JsonCleaner within TypeConverter
Posted by sb...@apache.org.
utilize improved JsonCleaner within TypeConverter
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/614f4034
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/614f4034
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/614f4034
Branch: refs/heads/master
Commit: 614f403448a6f19c563a75cf9cb2a8698d628cdc
Parents: 4e3d769
Author: sblackmon <sb...@apache.org>
Authored: Mon Aug 11 19:21:51 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:24 2014 -0500
----------------------------------------------------------------------
.../CleanAdditionalPropertiesProcessor.java | 2 +-
.../processor/DatasiftTypeConverterProcessor.java | 17 +----------------
2 files changed, 2 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/614f4034/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
index cc770db..00f08e7 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
@@ -48,7 +48,7 @@ public class CleanAdditionalPropertiesProcessor implements StreamsProcessor {
}
- public void cleanAdditionalProperties(ObjectNode node) {
+ public static void cleanAdditionalProperties(ObjectNode node) {
if( node.get("additionalProperties") != null ) {
ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
cleanAdditionalProperties(additionalProperties);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/614f4034/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
index 680c7ea..e85e8b1 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
@@ -115,20 +115,7 @@ public class DatasiftTypeConverterProcessor implements StreamsProcessor {
} else {
if(toConvert.getClass().equals(Activity.class)) { //hack to remove additional properties
ObjectNode node = mapper.convertValue(toConvert, ObjectNode.class);
- if(node.has("additionalProperties")) {
- ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
-// node.put("user_mentions", additionalProperties.get("user_mentions"));
- node.putAll(additionalProperties);
- node.remove("additionalProperties");
- }
- if(node.has("actor")) {
- ObjectNode actor = (ObjectNode) node.get("actor");
- if(actor.has("additionalProperties")) {
- ObjectNode additionalProperties = (ObjectNode) actor.get("additionalProperties");
- actor.putAll(additionalProperties);
- actor.remove("additionalProperties");
- }
- }
+ CleanAdditionalPropertiesProcessor.cleanAdditionalProperties(node);
return mapper.writeValueAsString(node);
} else
return mapper.writeValueAsString(toConvert);
@@ -139,8 +126,6 @@ public class DatasiftTypeConverterProcessor implements StreamsProcessor {
return null;
}
}
-
-
}
private class DefaultConverter implements DatasiftConverter {
[04/15] git commit: additionalProperties in serialized objects plague
this module so i'm adding a processor any implemention can use to fix it
Posted by sb...@apache.org.
additionalProperties in serialized objects plague this module
so i'm adding a processor any implemention can use to fix it
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/9acc2f05
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/9acc2f05
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/9acc2f05
Branch: refs/heads/master
Commit: 9acc2f054f35b489bbb136f5ee91acc8cc3e5bb5
Parents: d387b1d
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu Aug 7 17:41:32 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../CleanAdditionalPropertiesProcessor.java | 62 +++++++
.../DatasiftTypeConverterProcessor.java | 168 +++++++++++++++++++
.../DatasiftTypeConverterProcessor.java | 167 ------------------
.../DatasiftTypeConverterProcessorTest.java | 1 +
4 files changed, 231 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9acc2f05/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
new file mode 100644
index 0000000..cc770db
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
@@ -0,0 +1,62 @@
+package org.apache.streams.datasift.processor;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
+import com.google.common.collect.Lists;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * HACK PROCESSOR. Changes need to be made in apache streams to fix this issue long term.
+ */
+public class CleanAdditionalPropertiesProcessor implements StreamsProcessor {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(CleanAdditionalPropertiesProcessor.class);
+
+ private static final String ADDITIONAL_PROPERTIES = "additionalProperties";
+ private static final String EXTENSIONS = "extensions";
+
+ private ObjectMapper mapper;
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum datum) {
+ List<StreamsDatum> result = Lists.newLinkedList();
+ ObjectNode activity = this.mapper.convertValue(datum.getDocument(), ObjectNode.class);
+ cleanAdditionalProperties(activity);
+ datum.setDocument(activity);
+ result.add(datum);
+ return result;
+ }
+
+ @Override
+ public void prepare(Object o) {
+ this.mapper = StreamsJacksonMapper.getInstance();
+ this.mapper.registerModule(new JsonOrgModule());
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ public void cleanAdditionalProperties(ObjectNode node) {
+ if( node.get("additionalProperties") != null ) {
+ ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
+ cleanAdditionalProperties(additionalProperties);
+ Iterator<Map.Entry<String, JsonNode>> jsonNodeIterator = additionalProperties.fields();
+ while( jsonNodeIterator.hasNext() ) {
+ Map.Entry<String, JsonNode> entry = jsonNodeIterator.next();
+ node.put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9acc2f05/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
new file mode 100644
index 0000000..680c7ea
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
@@ -0,0 +1,168 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+package org.apache.streams.datasift.processor;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.datasift.Datasift;
+import org.apache.streams.datasift.provider.DatasiftConverter;
+import org.apache.streams.datasift.serializer.DatasiftActivitySerializer;
+import org.apache.streams.datasift.util.StreamsDatasiftMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ *
+ */
+public class DatasiftTypeConverterProcessor implements StreamsProcessor {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftTypeConverterProcessor.class);
+
+ private ObjectMapper mapper;
+ private Class outClass;
+ private DatasiftActivitySerializer datasiftInteractionActivitySerializer;
+ private DatasiftConverter converter;
+
+ public final static String TERMINATE = new String("TERMINATE");
+
+ public DatasiftTypeConverterProcessor(Class outClass) {
+ this.outClass = outClass;
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ List<StreamsDatum> result = Lists.newLinkedList();
+ Object doc;
+ try {
+ doc = this.converter.convert(entry.getDocument(), this.mapper);
+ if(doc != null) {
+ result.add(new StreamsDatum(doc, entry.getId()));
+ }
+ } catch (Exception e) {
+ LOGGER.error("Exception converting Datasift Interaction to "+this.outClass.getName()+ " : {}", e);
+ }
+ return result;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ this.mapper = StreamsDatasiftMapper.getInstance();
+ this.datasiftInteractionActivitySerializer = new DatasiftActivitySerializer();
+ if(this.outClass.equals(Activity.class)) {
+ this.converter = new ActivityConverter();
+ } else if (this.outClass.equals(String.class)) {
+ this.converter = new StringConverter();
+ } else {
+ LOGGER.warn("Using defaulting datasift converter");
+ this.converter = new DefaultConverter(this.outClass);
+ }
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ private class ActivityConverter implements DatasiftConverter {
+
+ @Override
+ public Object convert(Object toConvert, ObjectMapper mapper) {
+ if(toConvert instanceof Activity)
+ return toConvert;
+ try {
+ if(toConvert instanceof String)
+ return datasiftInteractionActivitySerializer.deserialize((String) toConvert);
+ return mapper.convertValue(toConvert, Activity.class);
+ } catch (Exception e) {
+ LOGGER.error("Exception while trying to convert {} to a Activity.", toConvert.getClass());
+ LOGGER.error("Exception : {}", e);
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+
+ }
+
+ private class StringConverter implements DatasiftConverter {
+ @Override
+ public Object convert(Object toConvert, ObjectMapper mapper) {
+ try {
+ if(toConvert instanceof String){
+ return mapper.writeValueAsString(mapper.readValue((String) toConvert, Datasift.class));
+ } else {
+ if(toConvert.getClass().equals(Activity.class)) { //hack to remove additional properties
+ ObjectNode node = mapper.convertValue(toConvert, ObjectNode.class);
+ if(node.has("additionalProperties")) {
+ ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
+// node.put("user_mentions", additionalProperties.get("user_mentions"));
+ node.putAll(additionalProperties);
+ node.remove("additionalProperties");
+ }
+ if(node.has("actor")) {
+ ObjectNode actor = (ObjectNode) node.get("actor");
+ if(actor.has("additionalProperties")) {
+ ObjectNode additionalProperties = (ObjectNode) actor.get("additionalProperties");
+ actor.putAll(additionalProperties);
+ actor.remove("additionalProperties");
+ }
+ }
+ return mapper.writeValueAsString(node);
+ } else
+ return mapper.writeValueAsString(toConvert);
+ }
+ } catch (Exception e) {
+ LOGGER.error("Exception while trying to write {} as a String.", toConvert.getClass());
+ LOGGER.error("Exception : {}", e);
+ return null;
+ }
+ }
+
+
+ }
+
+ private class DefaultConverter implements DatasiftConverter {
+
+ private Class clazz;
+
+ public DefaultConverter(Class clazz) {
+ this.clazz = clazz;
+ }
+
+ @Override
+ public Object convert(Object toConvert, ObjectMapper mapper) {
+ try {
+ if(toConvert instanceof String) {
+ return mapper.readValue((String) toConvert, this.clazz);
+ } else {
+ return mapper.convertValue(toConvert, this.clazz);
+ }
+
+ } catch (Exception e) {
+ throw new RuntimeException("Failed converting +"+ toConvert.getClass().getName()+" to "+ this.clazz.getName());
+ }
+ }
+ }
+};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9acc2f05/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessor.java
deleted file mode 100644
index 0b847a4..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessor.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-package org.apache.streams.datasift.provider;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Lists;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.serializer.DatasiftActivitySerializer;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- *
- */
-public class DatasiftTypeConverterProcessor implements StreamsProcessor {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftTypeConverterProcessor.class);
-
- private ObjectMapper mapper;
- private Class outClass;
- private DatasiftActivitySerializer datasiftInteractionActivitySerializer;
- private DatasiftConverter converter;
-
- public final static String TERMINATE = new String("TERMINATE");
-
- public DatasiftTypeConverterProcessor(Class outClass) {
- this.outClass = outClass;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- List<StreamsDatum> result = Lists.newLinkedList();
- Object doc;
- try {
- doc = this.converter.convert(entry.getDocument(), this.mapper);
- if(doc != null) {
- result.add(new StreamsDatum(doc, entry.getId()));
- }
- } catch (Exception e) {
- LOGGER.error("Exception converting Datasift Interaction to "+this.outClass.getName()+ " : {}", e);
- }
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.mapper = StreamsDatasiftMapper.getInstance();
- this.datasiftInteractionActivitySerializer = new DatasiftActivitySerializer();
- if(this.outClass.equals(Activity.class)) {
- this.converter = new ActivityConverter();
- } else if (this.outClass.equals(String.class)) {
- this.converter = new StringConverter();
- } else {
- LOGGER.warn("Using defaulting datasift converter");
- this.converter = new DefaultConverter(this.outClass);
- }
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- private class ActivityConverter implements DatasiftConverter {
-
- @Override
- public Object convert(Object toConvert, ObjectMapper mapper) {
- if(toConvert instanceof Activity)
- return toConvert;
- try {
- if(toConvert instanceof String)
- return datasiftInteractionActivitySerializer.deserialize((String) toConvert);
- return mapper.convertValue(toConvert, Activity.class);
- } catch (Exception e) {
- LOGGER.error("Exception while trying to convert {} to a Activity.", toConvert.getClass());
- LOGGER.error("Exception : {}", e);
- e.printStackTrace();
- return null;
- }
- }
-
-
- }
-
- private class StringConverter implements DatasiftConverter {
- @Override
- public Object convert(Object toConvert, ObjectMapper mapper) {
- try {
- if(toConvert instanceof String){
- return mapper.writeValueAsString(mapper.readValue((String) toConvert, Datasift.class));
- } else {
- if(toConvert.getClass().equals(Activity.class)) { //hack to remove additional properties
- ObjectNode node = mapper.convertValue(toConvert, ObjectNode.class);
- if(node.has("additionalProperties")) {
- ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
-// node.put("user_mentions", additionalProperties.get("user_mentions"));
- node.putAll(additionalProperties);
- node.remove("additionalProperties");
- }
- if(node.has("actor")) {
- ObjectNode actor = (ObjectNode) node.get("actor");
- if(actor.has("additionalProperties")) {
- ObjectNode additionalProperties = (ObjectNode) actor.get("additionalProperties");
- actor.putAll(additionalProperties);
- actor.remove("additionalProperties");
- }
- }
- return mapper.writeValueAsString(node);
- } else
- return mapper.writeValueAsString(toConvert);
- }
- } catch (Exception e) {
- LOGGER.error("Exception while trying to write {} as a String.", toConvert.getClass());
- LOGGER.error("Exception : {}", e);
- return null;
- }
- }
-
-
- }
-
- private class DefaultConverter implements DatasiftConverter {
-
- private Class clazz;
-
- public DefaultConverter(Class clazz) {
- this.clazz = clazz;
- }
-
- @Override
- public Object convert(Object toConvert, ObjectMapper mapper) {
- try {
- if(toConvert instanceof String) {
- return mapper.readValue((String) toConvert, this.clazz);
- } else {
- return mapper.convertValue(toConvert, this.clazz);
- }
-
- } catch (Exception e) {
- throw new RuntimeException("Failed converting +"+ toConvert.getClass().getName()+" to "+ this.clazz.getName());
- }
- }
- }
-};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9acc2f05/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java
index 7d44e7f..fac0f02 100644
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftTypeConverterProcessorTest.java
@@ -20,6 +20,7 @@ package org.apache.streams.datasift.provider;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.datasift.processor.DatasiftTypeConverterProcessor;
import org.apache.streams.pojo.json.Activity;
import org.junit.Test;
[09/15] git commit: making setup method non-static again
Posted by sb...@apache.org.
making setup method non-static again
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/1360223d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/1360223d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/1360223d
Branch: refs/heads/master
Commit: 1360223d93c458412ac6b4b3c36b5ba069cb693b
Parents: e6380ce
Author: sblackmon <sb...@w2odigital.com>
Authored: Fri Aug 8 15:25:21 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../streams/datasift/provider/DatasiftManagedSourceSetup.java | 3 ++-
.../streams/datasift/provider/DatasiftStreamConfigurator.java | 6 +++++-
.../streams/datasift/provider/DatasiftStreamProvider.java | 2 +-
3 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1360223d/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
index 6e648e7..03bd7d1 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
@@ -54,7 +54,8 @@ public class DatasiftManagedSourceSetup implements Runnable {
}
public void setup() {
- client = DatasiftStreamProvider.getNewClient(config.getUserName(), config.getApiKey());
+
+ client = new DatasiftStreamProvider(null, config).getNewClient(config.getUserName(), config.getApiKey());
}
public void current() {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1360223d/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
index 5726b4f..07716a1 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
@@ -18,9 +18,11 @@ under the License.
*/
package org.apache.streams.datasift.provider;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigRenderOptions;
import org.apache.streams.datasift.DatasiftConfiguration;
+import org.apache.streams.datasift.util.StreamsDatasiftMapper;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,12 +34,14 @@ public class DatasiftStreamConfigurator {
private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamConfigurator.class);
+ private static final ObjectMapper MAPPER = StreamsDatasiftMapper.getInstance();
+
public static DatasiftConfiguration detectConfiguration(Config datasift) {
DatasiftConfiguration datasiftConfiguration = null;
try {
- datasiftConfiguration = mapper.readValue(datasift.root().render(ConfigRenderOptions.concise()), DatasiftConfiguration.class);
+ datasiftConfiguration = MAPPER.readValue(datasift.root().render(ConfigRenderOptions.concise()), DatasiftConfiguration.class);
} catch (Exception e) {
e.printStackTrace();
LOGGER.warn("Could not parse datasiftConfiguration");
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1360223d/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
index c3f5ea3..8ed1443 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
@@ -128,7 +128,7 @@ public class DatasiftStreamProvider implements StreamsProvider {
* @param apiKey
* @return
*/
- protected static DataSiftClient getNewClient(String userName, String apiKey) {
+ protected DataSiftClient getNewClient(String userName, String apiKey) {
return new DataSiftClient(new DataSiftConfig(userName, apiKey));
}
[03/15] git commit: instagram support
Posted by sb...@apache.org.
instagram support
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e8c85ee8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e8c85ee8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e8c85ee8
Branch: refs/heads/master
Commit: e8c85ee85a2ea1e343728913f42f747a8d6aa408
Parents: f0404ba
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu Aug 7 13:56:09 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../DatasiftActivitySerializerTest.java | 12 ++++++++++
.../test/resources/instagram_datasift_json.txt | 25 ++++++++++++++++++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e8c85ee8/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
index 90b7285..daff2d7 100644
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
@@ -49,6 +49,18 @@ public class DatasiftActivitySerializerTest {
}
}
+ @Test
+ public void testInstagramConversion() throws Exception {
+ Scanner scanner = new Scanner(DatasiftActivitySerializerTest.class.getResourceAsStream("/instagram_datasift_json.txt"));
+ String line = null;
+ while(scanner.hasNextLine()) {
+ line = scanner.nextLine();
+ testGeneralConversion(line);
+ System.out.println("ORIGINAL -> "+line);
+ System.out.println("ACTIVITY -> "+MAPPER.writeValueAsString(SERIALIZER.deserialize(line)));
+ System.out.println("NODE -> "+MAPPER.convertValue(SERIALIZER.deserialize(line), JsonNode.class));
+ }
+ }
/**
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e8c85ee8/streams-contrib/streams-provider-datasift/src/test/resources/instagram_datasift_json.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/resources/instagram_datasift_json.txt b/streams-contrib/streams-provider-datasift/src/test/resources/instagram_datasift_json.txt
new file mode 100644
index 0000000..91b5164
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/test/resources/instagram_datasift_json.txt
@@ -0,0 +1,25 @@
+{"demographic":{"gender":"male"},"instagram":{"created_at":"Tue, 05 Aug 2014 17:27:40 +0100","created_time":"1407256060","from":{"full_name":"Victor Miranda","id":"298295070","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_298295070_75sq_1368841581.jpg","username":"victormiranda23"},"id":"780442861925870780","media":{"caption":"Who fancies a donut? #assen","created_time":1407222356,"filter":"Rise","id":"780160131451366601_476322","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpf1\/t51.2885-15\/10544278_1449673285293147_1305566037_n.jpg","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","tags":["assen"],"type":"image","username":"redbull","users_in_photo":["redbullracing","sebastien_buemi"]},"text":"@fusion_shoutz got me 1k","type":"comment"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_298295070_75sq_1368841581.jpg","id":"298295070","link":"http:\/\/instagram.com\/victormiranda23","name":"Victor Miranda","usern
ame":"victormiranda23"},"content":"@fusion_shoutz got me 1k","created_at":"Tue, 05 Aug 2014 17:27:40 +0100","id":"1e41cbd6b25ea600e019000070277007","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","subtype":"comment","type":"instagram"},"language":{"confidence":90,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"created_at":"Tue, 05 Aug 2014 17:43:45 +0100","created_time":"1407257025","from":{"full_name":"G\u0101rr\u00EBtt D\u012Fx\u00F8\u0144","id":"146294212","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_146294212_75sq_1369977373.jpg","username":"g_dixon94"},"id":"780450951807151844","media":{"caption":"Who fancies a donut? #assen","created_time":1407222356,"filter":"Rise","id":"780160131451366601_476322","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpf1\/t51.2885-15\/10544278_1449673285293147_1305566037_n.jpg","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","tags":["assen"],"type":"image","username":"redbull","users_in_photo":["redbullracing","sebastien_buemi"]},"text":"Yes vettels 2014 world chanpionhip winning car(one of the many)","type":"comment"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_146294212_75sq_1369977373.jpg","id":"146294212","link":"http:\/\/instagram.com\/g_dixon94","name":"G\u
0101rr\u00EBtt D\u012Fx\u00F8\u0144","username":"g_dixon94"},"content":"Yes vettels 2014 world chanpionhip winning car(one of the many)","created_at":"Tue, 05 Aug 2014 17:43:45 +0100","id":"1e41cbfaa553a680e019ffffe184b7f5","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","received_at":4743689271810658927,"subtype":"comment","type":"instagram"},"language":{"confidence":98,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"created_at":"Tue, 05 Aug 2014 17:43:58 +0100","created_time":"1407257038","from":{"full_name":"G\u0101rr\u00EBtt D\u012Fx\u00F8\u0144","id":"146294212","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_146294212_75sq_1369977373.jpg","username":"g_dixon94"},"id":"780451061504978668","media":{"caption":"Who fancies a donut? #assen","created_time":1407222356,"filter":"Rise","id":"780160131451366601_476322","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpf1\/t51.2885-15\/10544278_1449673285293147_1305566037_n.jpg","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","tags":["assen"],"type":"image","username":"redbull","users_in_photo":["redbullracing","sebastien_buemi"]},"text":"@_squirrel30shenk_","type":"comment"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_146294212_75sq_1369977373.jpg","id":"146294212","link":"http:\/\/instagram.com\/g_dixon94","name":"G\u0101rr\u00EBtt D\u012Fx\u00F8\u0144","usernam
e":"g_dixon94"},"content":"@_squirrel30shenk_","created_at":"Tue, 05 Aug 2014 17:43:58 +0100","id":"1e41cbfb214dab00e019ffffd1f3ccfe","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","received_at":4743689271976866611,"subtype":"comment","type":"instagram"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"mostly_male"},"instagram":{"created_at":"Tue, 05 Aug 2014 17:52:45 +0100","created_time":"1407257565","from":{"full_name":"Jos\u00E9 Samman","id":"1446048106","profile_picture":"http:\/\/photos-g.ak.instagram.com\/hphotos-ak-xpa1\/10554254_599380053512358_34771882_a.jpg","username":"j_samman"},"id":"780455485572951057","media":{"caption":"Who fancies a donut? #assen","created_time":1407222356,"filter":"Rise","id":"780160131451366601_476322","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpf1\/t51.2885-15\/10544278_1449673285293147_1305566037_n.jpg","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","tags":["assen"],"type":"image","username":"redbull","users_in_photo":["redbullracing","sebastien_buemi"]},"text":"@fusion_shoutz got me 1k","type":"comment"},"interaction":{"author":{"avatar":"http:\/\/photos-g.ak.instagram.com\/hphotos-ak-xpa1\/10554254_599380053512358_34771882_a.jpg","id":"1446048106","link":"http:\/\/instagram.com\/j_samman","name":"Jos
\u00E9 Samman","username":"j_samman"},"content":"@fusion_shoutz got me 1k","created_at":"Tue, 05 Aug 2014 17:52:45 +0100","id":"1e41cc0ec32aac80e019000012f1b4d4","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","subtype":"comment","type":"instagram"},"language":{"confidence":90,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"created_at":"Tue, 05 Aug 2014 18:01:57 +0100","created_time":"1407258117","from":{"full_name":"Gord","id":"178572119","profile_picture":"http:\/\/photos-e.ak.instagram.com\/hphotos-ak-xap1\/10413816_1442711092652044_270055028_a.jpg","username":"gordond_"},"id":"780460118374657342","media":{"caption":"Who fancies a donut? #assen","created_time":1407222356,"filter":"Rise","id":"780160131451366601_476322","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpf1\/t51.2885-15\/10544278_1449673285293147_1305566037_n.jpg","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","tags":["assen"],"type":"image","username":"redbull","users_in_photo":["redbullracing","sebastien_buemi"]},"text":"@james_bajc bad ass","type":"comment"},"interaction":{"author":{"avatar":"http:\/\/photos-e.ak.instagram.com\/hphotos-ak-xap1\/10413816_1442711092652044_270055028_a.jpg","id":"178572119","link":"http:\/\/instagram.com\/gordond_","name":"Gord","username":"gordond_"},"content":"@james_bajc bad a
ss","created_at":"Tue, 05 Aug 2014 18:01:57 +0100","id":"1e41cc235373a080e01900003bee88e6","link":"http:\/\/instagram.com\/p\/rTr4GPFKzJ\/","received_at":4743689276357561287,"subtype":"comment","type":"instagram"},"language":{"confidence":88,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"from":{"full_name":"edgar_merlin","id":"1150604739","profile_picture":"http:\/\/photos-a.ak.instagram.com\/hphotos-ak-xpa1\/10537151_614415332005328_1203218744_a.jpg","username":"edgar_merlin"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/photos-a.ak.instagram.com\/hphotos-ak-xpa1\/10537151_614415332005328_1203218744_a.jpg","id":"1150604739","link":"http:\/
\/instagram.com\/edgar_merlin","name":"edgar_merlin","username":"edgar_merlin"},"content":"edgar_merlin likes pang_miss's image","created_at":"Wed, 06 Aug 2014 04:10:46 +0100","id":"1e41cc3200eda180e01900000c0542f5","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689429558700802,"subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"unisex"},"instagram":{"from":{"full_name":"Michele Tomizioli","id":"30130462","profile_picture":"http:\/\/photos-f.ak.instagram.com\/hphotos-ak-xpa1\/10354459_570400436406085_1614045732_a.jpg","username":"tomizioli84"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/photos-f.ak.instagram.com\/hphotos-ak-xpa1\/10354459_570400436406085_1614045732_a.jp
g","id":"30130462","link":"http:\/\/instagram.com\/tomizioli84","name":"Michele Tomizioli","username":"tomizioli84"},"content":"tomizioli84 likes pang_miss's image","created_at":"Wed, 06 Aug 2014 03:32:24 +0100","id":"1e41cc3200eda180e01900001056a4c5","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689419902041457,"subtype":"like","type":"instagram"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"from":{"full_name":"bibierassa","id":"283406228","profile_picture":"http:\/\/photos-e.ak.instagram.com\/hphotos-ak-xfp1\/10560985_1455693928035428_1056730897_a.jpg","username":"bibierassa"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/photos-e.ak.instagram.com\/hphotos-ak-xfp1\/10560985_1455693928035428_1056730897_a.jpg","id":"283406228","link":"http:\/\/in
stagram.com\/bibierassa","name":"bibierassa","username":"bibierassa"},"content":"bibierassa likes pang_miss's image","created_at":"Thu, 07 Aug 2014 06:40:22 +0100","id":"1e41cc3200eda180e019000010c6fe88","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"fi","tag_extended":"fi"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"male"},"instagram":{"from":{"full_name":"Eric Chong","id":"30701373","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_30701373_75sq_1359382742.jpg","username":"chongchunyin"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_30701373_75sq_1359382742.jpg","id":"30701373","link":"http:\/\/
instagram.com\/chongchunyin","name":"Eric Chong","username":"chongchunyin"},"content":"chongchunyin likes pang_miss's image","created_at":"Wed, 06 Aug 2014 03:43:15 +0100","id":"1e41cc3200eda180e01900001a69d8a7","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","subtype":"like","type":"instagram"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"caption":{"created_time":"1407258511","from":{"full_name":"pang_miss","id":"24094386","profile_picture":"http:\/\/photos-h.ak.instagram.com\/hphotos-ak-xap1\/928614_1632022843690159_35582184_a.jpg","username":"pang_miss"},"id":"780463416711352313","text":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday"},"created_at":"Tue, 05 Aug 2014 18:08:31 +0100","created_time":"1407258511","filter":"Amaro","from":{"bio":"","full_name":"pang_miss","id":"24094386","profile_picture":"http:\/\/photos-h.ak.instagram.com\/hphotos-ak-xap1\/928614_1632022843690159_35582184_a.jpg","username":"pang_miss","website":""},"geo":{"latitude":4630278815558678834,"longitude":-4585567004724296657},"id":"780463416006708768_24094386","images":{"low_resolution":{"height":306,"url":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_a.jpg","width":306},"standard_r
esolution":{"height":640,"url":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","width":640},"thumbnail":{"height":150,"url":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_s.jpg","width":150}},"link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","location":{"id":866213,"name":"The Cosmopolitan of Las Vegas"},"tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image"},"interaction":{"author":{"avatar":"http:\/\/photos-h.ak.instagram.com\/hphotos-ak-xap1\/928614_1632022843690159_35582184_a.jpg","id":"24094386","link":"http:\/\/instagram.com\/pang_miss","name":"pang_miss","username":"pang_miss"},"content":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_at":"T
ue, 05 Aug 2014 18:08:31 +0100","geo":{"latitude":36.109769391,"longitude":-115.173509039},"hashtags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"id":"1e41cc3200eda180e019000038ebe702","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689278022486065,"subtype":"image","type":"instagram"},"language":{"confidence":90,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"from":{"full_name":"Plengpernn","id":"307434701","profile_picture":"http:\/\/photos-f.ak.instagram.com\/hphotos-ak-xpf1\/10448989_284780761694189_1438378141_a.jpg","username":"plengpernn"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/photos-f.ak.instagram.com\/hphotos-ak-xpf1\/10448989_284780761694189_1438378141_a.jpg","id":"307434701","link":"http:\/\/inst
agram.com\/plengpernn","name":"Plengpernn","username":"plengpernn"},"content":"plengpernn likes pang_miss's image","created_at":"Thu, 07 Aug 2014 07:46:03 +0100","id":"1e41cc3200eda180e019000039c43add","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"male"},"instagram":{"from":{"full_name":"Timothy A. Slater \u2705\uE03D\uE12A\u00A9","id":"174524333","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_174524333_75sq_1398139139.jpg","username":"therealtimothyslater"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_174524333_75sq_139813
9139.jpg","id":"174524333","link":"http:\/\/instagram.com\/therealtimothyslater","name":"Timothy A. Slater \u2705\uE03D\uE12A\u00A9","username":"therealtimothyslater"},"content":"therealtimothyslater likes pang_miss's image","created_at":"Tue, 05 Aug 2014 20:16:52 +0100","id":"1e41cc3200eda180e01900004b578617","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"female"},"instagram":{"from":{"full_name":"Ana Paula Ferreira\u2728\uD83C\uDF1F\uD83D\uDCAB","id":"311234315","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_311234315_75sq_1385001890.jpg","username":"11aninha"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_311234315_75sq_1385001890
.jpg","id":"311234315","link":"http:\/\/instagram.com\/11aninha","name":"Ana Paula Ferreira\u2728\uD83C\uDF1F\uD83D\uDCAB","username":"11aninha"},"content":"11aninha likes pang_miss's image","created_at":"Tue, 05 Aug 2014 18:13:37 +0100","id":"1e41cc3200eda180e019000060d61d05","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","subtype":"like","type":"instagram"},"language":{"confidence":96,"tag":"pt","tag_extended":"pt"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"from":{"full_name":"\uD83D\uDCF2 248.495.7448 \u2611\uFE0F","id":"594131211","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_594131211_75sq_1394302424.jpg","username":"_jqs1oak"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_594131211_75sq_1394302424.jpg","id":"594131211","link":"http:\/\/inst
agram.com\/_jqs1oak","name":"\uD83D\uDCF2 248.495.7448 \u2611\uFE0F","username":"_jqs1oak"},"content":"_jqs1oak likes pang_miss's image","created_at":"Tue, 05 Aug 2014 21:42:56 +0100","id":"1e41cc3200eda180e01900006564d5b2","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","subtype":"like","type":"instagram"},"language":{"confidence":96,"tag":"eu","tag_extended":"eu"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"female"},"instagram":{"from":{"full_name":"Kali ishida","id":"1246334926","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_1246334926_75sq_1396811902.jpg","username":"viplasvegaskali"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_1246334926_75sq_1396811902.jpg","id":"1246334926","li
nk":"http:\/\/instagram.com\/viplasvegaskali","name":"Kali ishida","username":"viplasvegaskali"},"content":"viplasvegaskali likes pang_miss's image","created_at":"Tue, 05 Aug 2014 18:10:18 +0100","id":"1e41cc3200eda180e019000070d947cf","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689278444613403,"subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"no","tag_extended":"no"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"from":{"full_name":"pang_miss","id":"24094386","profile_picture":"http:\/\/photos-h.ak.instagram.com\/hphotos-ak-xap1\/928614_1632022843690159_35582184_a.jpg","username":"pang_miss"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/photos-h.ak.instagram.com\/hphotos-ak-xap1\/928614_1632022843690159_35582184_a.jpg","id":"24094386","link":"http:\/\/instagram.com\
/pang_miss","name":"pang_miss","username":"pang_miss"},"content":"pang_miss likes pang_miss's image","created_at":"Thu, 07 Aug 2014 14:39:02 +0100","id":"1e41cc3200eda180e01900007aaf7cf5","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689950056962589,"subtype":"like","type":"instagram"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"female"},"instagram":{"from":{"full_name":"Yukiko","id":"6971121","profile_picture":"http:\/\/photos-b.ak.instagram.com\/hphotos-ak-xpa1\/10475000_263173467202529_2114301415_a.jpg","username":"ririkochang"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/photos-b.ak.instagram.com\/hphotos-ak-xpa1\/10475000_263173467202529_2114301415_a.jpg","id":"697
1121","link":"http:\/\/instagram.com\/ririkochang","name":"Yukiko","username":"ririkochang"},"content":"ririkochang likes pang_miss's image","created_at":"Wed, 06 Aug 2014 01:03:46 +0100","id":"1e41cc3200eda180e01900007d5c03bb","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"female"},"instagram":{"from":{"full_name":"Lidia Bonilla","id":"419276466","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_419276466_75sq_1371298258.jpg","username":"themuabox"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_419276466_75sq_1371298258.jpg","id":"419276466","link":"htt
p:\/\/instagram.com\/themuabox","name":"Lidia Bonilla","username":"themuabox"},"content":"themuabox likes pang_miss's image","created_at":"Tue, 05 Aug 2014 18:10:18 +0100","id":"1e41cc3200eda180e019ffff9c86757d","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689278444613403,"subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"unisex"},"instagram":{"from":{"full_name":"S R L I M A","id":"317562133","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_317562133_75sq_1390788520.jpg","username":"srllima"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_317562133_75sq_1390788520.jpg","id":"317562133","link":"http:\
/\/instagram.com\/srllima","name":"S R L I M A","username":"srllima"},"content":"srllima likes pang_miss's image","created_at":"Tue, 05 Aug 2014 18:11:07 +0100","id":"1e41cc3200eda180e019ffffa3841e3a","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689278653254861,"subtype":"like","type":"instagram"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"mostly_male"},"instagram":{"from":{"full_name":"Ali","id":"383396910","profile_picture":"http:\/\/photos-d.ak.instagram.com\/hphotos-ak-xpa1\/10507828_308288519345947_313189792_a.jpg","username":"alialhadhrami7"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/photos-d.ak.instagram.com\/hphotos-ak-xpa1\/10507828_308288519345947_313189792_a.jpg","id"
:"383396910","link":"http:\/\/instagram.com\/alialhadhrami7","name":"Ali","username":"alialhadhrami7"},"content":"alialhadhrami7 likes pang_miss's image","created_at":"Wed, 06 Aug 2014 05:41:37 +0100","id":"1e41cc3200eda180e019ffffa3d5ef85","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689452423567376,"subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"tl","tag_extended":"tl"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"unisex"},"instagram":{"from":{"full_name":"Naif S. Alofi","id":"355096627","profile_picture":"http:\/\/photos-g.ak.instagram.com\/hphotos-ak-xaf1\/10584664_1657360314489254_1143597454_a.jpg","username":"naifalofi"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/photos-g.ak.instagram.com\/hphotos-ak-xaf1\/10584664_1657360314489254_1143597454_a.jpg",
"id":"355096627","link":"http:\/\/instagram.com\/naifalofi","name":"Naif S. Alofi","username":"naifalofi"},"content":"naifalofi likes pang_miss's image","created_at":"Tue, 05 Aug 2014 22:48:33 +0100","id":"1e41cc3200eda180e019ffffd0e01d66","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689348471328866,"subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"sm","tag_extended":"sm"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"from":{"full_name":"koskam_","id":"203451291","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_203451291_75sq_1396554171.jpg","username":"koskam_"},"media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_203451291_75sq_1396554171.jpg","id":"203451291","link":"http:\/\/instagram.com\/koskam_","name":"kosk
am_","username":"koskam_"},"content":"koskam_ likes pang_miss's image","created_at":"Wed, 06 Aug 2014 01:53:50 +0100","id":"1e41cc3200eda180e019ffffd20206d2","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","subtype":"like","type":"instagram"},"language":{"confidence":96,"tag":"fi","tag_extended":"fi"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"female"},"instagram":{"created_at":"Tue, 05 Aug 2014 18:10:09 +0100","created_time":"1407258609","from":{"full_name":"Kali ishida","id":"1246334926","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_1246334926_75sq_1396811902.jpg","username":"viplasvegaskali"},"id":"780464241470901298","media":{"caption":"Fucked up #cosmopolitan#vegas#goodvibes#nightclub#nightlife#urbanoutfitters#uo#short##awesome#cool#ilikesummer#summertime#relaxtime#relaxday","created_time":1407258511,"filter":"Amaro","id":"780463416006708768_24094386","image":"http:\/\/scontent-b.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/924436_326627004180028_2126561455_n.jpg","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","tags":["vegas","short","relaxday","summertime","awesome","ilikesummer","nightclub","uo","goodvibes","nightlife","urbanoutfitters","cosmopolitan","relaxtime","cool"],"type":"image","username":"pang_miss"},"text":"@pang_miss Hey, if u arent already set up and
would like free guestlist to dayclubs & nightclubs, text me 702-817-5477 :) have fun!","type":"comment"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_1246334926_75sq_1396811902.jpg","id":"1246334926","link":"http:\/\/instagram.com\/viplasvegaskali","name":"Kali ishida","username":"viplasvegaskali"},"content":"@pang_miss Hey, if u arent already set up and would like free guestlist to dayclubs & nightclubs, text me 702-817-5477 :) have fun!","created_at":"Tue, 05 Aug 2014 18:10:09 +0100","id":"1e41cc35a786ae80e019ffffd3d989c3","link":"http:\/\/instagram.com\/p\/rUw1d_RI4g\/","received_at":4743689278444613403,"subtype":"comment","type":"instagram"},"language":{"confidence":99,"tag":"en","tag_extended":"en"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"demographic":{"gender":"mostly_female"},"instagram":{"from":{"full_name":"The Original K. Michelle \u2764\uFE0F\uD83D\uDC18","id":"243104122","profile_picture":"http:\/\/photos-f.ak.instagram.com\/hphotos-ak-xfa1\/10546582_1430099920610837_923379227_a.jpg","username":"ricekrissytreat"},"media":{"caption":"a giant taste of summer. \u2600\uFE0F","created_time":1407258680,"filter":"Valencia","id":"780464841004697308_18169368","image":"http:\/\/scontent-a.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/10518268_816493021729059_2045569609_n.jpg","link":"http:\/\/instagram.com\/p\/rUxKNHxEbc\/","tags":["wingsteam","ilikesummer","redbull","summeredition","redbullgirls"],"type":"image","username":"torae__"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/photos-f.ak.instagram.com\/hphotos-ak-xfa1\/10546582_1430099920610837_923379227_a.jpg","id":"243104122","link":"http:\/\/instagram.com\/ricekrissytreat","name":"The Original K. Michelle \u2764\uFE0F\uD83D\uDC18","username":"ricekr
issytreat"},"content":"ricekrissytreat likes torae__'s image","created_at":"Tue, 05 Aug 2014 18:13:37 +0100","id":"1e41cc384ca2ac00e01900000d78bf09","link":"http:\/\/instagram.com\/p\/rUxKNHxEbc\/","subtype":"like","type":"instagram"},"language":{"confidence":97,"tag":"da","tag_extended":"da"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
+{"instagram":{"from":{"full_name":"millhouse84","id":"46327078","profile_picture":"http:\/\/images.ak.instagram.com\/profiles\/profile_46327078_75sq_1383148997.jpg","username":"millhouse84"},"media":{"caption":"a giant taste of summer. \u2600\uFE0F","created_time":1407258680,"filter":"Valencia","id":"780464841004697308_18169368","image":"http:\/\/scontent-a.cdninstagram.com\/hphotos-xpa1\/t51.2885-15\/10518268_816493021729059_2045569609_n.jpg","link":"http:\/\/instagram.com\/p\/rUxKNHxEbc\/","tags":["wingsteam","ilikesummer","redbull","summeredition","redbullgirls"],"type":"image","username":"torae__"},"type":"like"},"interaction":{"author":{"avatar":"http:\/\/images.ak.instagram.com\/profiles\/profile_46327078_75sq_1383148997.jpg","id":"46327078","link":"http:\/\/instagram.com\/millhouse84","name":"millhouse84","username":"millhouse84"},"content":"millhouse84 likes torae__'s image","created_at":"Tue, 05 Aug 2014 20:30:15 +0100","id":"1e41cc384ca2ac00e019000010c4e523","link":"http:\
/\/instagram.com\/p\/rUxKNHxEbc\/","subtype":"like","type":"instagram"},"language":{"confidence":96,"tag":"da","tag_extended":"da"},"source":{"id":"a536aa0bac1f41e2baf5d854d36a02be"}}
[12/15] broke channels within Datasift.json out into fragments
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json
new file mode 100644
index 0000000..9a0f027
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json
@@ -0,0 +1,133 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType": "org.apache.streams.datasift.DatasiftConfiguration",
+ "javaInterfaces": [
+ "java.io.Serializable"
+ ],
+ "properties": {
+ "apiKey": {
+ "type": "string"
+ },
+ "userName": {
+ "type": "string"
+ },
+ "streamHash": {
+ "type": "array",
+ "minItems": 0,
+ "items": {
+ "type": "string"
+ }
+ },
+ "streamCsdl": {
+ "type": "array",
+ "minItems": 0,
+ "items": {
+ "type": "string"
+ }
+ },
+ "managedSources": {
+ "type": "array",
+ "minItems": 0,
+ "items": {
+ "javaType": "org.apache.streams.datasift.managed.StreamsManagedSource",
+ "type": "object",
+ "properties": {
+ "auth": {
+ "type": "array",
+ "items": {
+ "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceAuth",
+ "type": "object",
+ "properties": {
+ "identity_id": {
+ "type": "string"
+ },
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "value": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ },
+ "created_at": {
+ "type": "number"
+ },
+ "id": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "parameters": {
+ "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceParams",
+ "type": "object",
+ "properties": {
+ "comments": {
+ "type": "boolean"
+ },
+ "likes": {
+ "type": "boolean"
+ }
+ }
+ },
+ "resources": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceResource",
+ "type": "object",
+ "properties": {
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "distance": {
+ "type": "number"
+ },
+ "exact_match": {
+ "type": "string"
+ },
+ "foursq": {
+ "type": "string"
+ },
+ "lat": {
+ "type": "number"
+ },
+ "lng": {
+ "type": "number"
+ },
+ "type": {
+ "type": "string",
+ "enum" : ["user", "tag", "area", "location", "popular"]
+ },
+ "usermeta": {
+ "type": "string"
+ },
+ "value": {
+ "type": "number"
+ }
+ }
+ },
+ "resource_id": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "source_type": {
+ "type": "string",
+ "enum" : ["facebook_page", "googleplus", "instagram", "yammer"]
+ },
+ "status": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json
new file mode 100644
index 0000000..bb65ef0
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json
@@ -0,0 +1,17 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.datasift.DatasiftPushConfiguration",
+ "extends": {"$ref":"DatasiftConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "streamHash": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json
new file mode 100644
index 0000000..91a9974
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json
@@ -0,0 +1,17 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.datasift.DatasiftStreamConfiguration",
+ "extends": {"$ref":"DatasiftConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "streamHash": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json
new file mode 100644
index 0000000..a483d79
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json
@@ -0,0 +1,120 @@
+{
+ "type" : "object",
+ "$schema" : "http://json-schema.org/draft-03/schema",
+ "javaType" : "org.apache.streams.datasift.facebook.Facebook",
+ "properties" :
+ {
+ "application" :
+ {
+ "type" : "string"
+ },
+ "author" :
+ {
+ "javaType" : "org.apache.streams.datasift.facebook.Author",
+ "type" : "object",
+ "dynamic" : "true",
+ "properties" :
+ {
+ "avatar" :
+ {
+ "type" : "string"
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "link" :
+ {
+ "type" : "string"
+ },
+ "name" :
+ {
+ "type" : "string"
+ }
+ }
+ },
+ "created_at" :
+ {
+ "type" : "string",
+ "format" : "date-time"
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "likes" :
+ {
+ "javaType" : "org.apache.streams.datasift.facebook.Likes",
+ "type" : "object",
+ "dynamic" : "true",
+ "properties" :
+ {
+ "count" :
+ {
+ "type" : "integer"
+ },
+ "ids" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "names" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ }
+ }
+ },
+ "message" :
+ {
+ "type" : "string"
+ },
+ "source" :
+ {
+ "type" : "string"
+ },
+ "to" :
+ {
+ "javaType" : "org.apache.streams.datasift.facebook.To",
+ "type" : "object",
+ "dynamic" : "true",
+ "properties" :
+ {
+ "ids" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "names" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ }
+ }
+ },
+ "type" :
+ {
+ "type" : "string"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json
new file mode 100644
index 0000000..1397cba
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json
@@ -0,0 +1,178 @@
+{
+ "type" : "object",
+ "$schema" : "http://json-schema.org/draft-03/schema",
+ "javaType" : "org.apache.streams.datasift.instagram.Instagram",
+ "properties" :
+ {
+ "caption" :
+ {
+ "type" : "object",
+ "properties" :
+ {
+ "created_time" :
+ {
+ "type" : "number"
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "text" :
+ {
+ "type" : "string"
+ },
+ "from" :
+ {
+ "type" : "object",
+ "properties" :
+ {
+ "full_name" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "profile_picture" :
+ {
+ "type" : "string"
+ },
+ "username" :
+ {
+ "type" : "string"
+ }
+ }
+ }
+ }
+ },
+ "created_at" :
+ {
+ "type" : "string",
+ "format" : "date-time"
+ },
+ "from" :
+ {
+ "javaType" : "org.apache.streams.datasift.instagram.From",
+ "type" : "object",
+ "properties" :
+ {
+ "full_name" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "profile_picture" :
+ {
+ "type" : "string"
+ },
+ "username" :
+ {
+ "type" : "string"
+ }
+ }
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "images" :
+ {
+ "type" : "object",
+ "properties" :
+ {
+ "low_resolution" :
+ {
+ "type" : "object"
+ },
+ "standard_resolution" :
+ {
+ "type" : "object"
+ },
+ "thumbnail" :
+ {
+ "type" : "object"
+ }
+ }
+ },
+ "media" :
+ {
+ "javaType" : "org.apache.streams.datasift.instagram.Media",
+ "type" : "object",
+ "properties" :
+ {
+ "caption" :
+ {
+ "type" : "string"
+ },
+ "created_time" :
+ {
+ "type" : "number"
+ },
+ "filter" :
+ {
+ "type" : "string"
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "image" :
+ {
+ "type" : "string"
+ },
+ "link" :
+ {
+ "type" : "string"
+ },
+ "tags" :
+ {
+ "type" : "array",
+ "items" :
+ {
+ "type" : "string"
+ }
+ },
+ "type" :
+ {
+ "type" : "string"
+ },
+ "username" :
+ {
+ "type" : "string"
+ },
+ "video" :
+ {
+ "type" : "string"
+ }
+ }
+ },
+ "tags" :
+ {
+ "type" : "array",
+ "items" :
+ {
+ "type" : "string"
+ }
+ },
+ "type" :
+ {
+ "type" : "string"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/interaction/DatasiftInteraction.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/interaction/DatasiftInteraction.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/interaction/DatasiftInteraction.json
new file mode 100644
index 0000000..fd4d576
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/interaction/DatasiftInteraction.json
@@ -0,0 +1,92 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "javaType": "org.apache.streams.datasift.interaction.Interaction",
+ "properties": {
+ "author": {
+ "javaType": "org.apache.streams.datasift.interaction.Author",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "avatar": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "link": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "content": {
+ "type": "string"
+ },
+ "contenttype": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string",
+ "format": "date-time"
+ },
+ "geo": {
+ "javaType": "org.apache.streams.datasift.interaction.Geo",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "latitude": {
+ "type": "number"
+ },
+ "longitude": {
+ "type": "number"
+ }
+ }
+ },
+ "id": {
+ "type": "string"
+ },
+ "link": {
+ "type": "string"
+ },
+ "received_at": {
+ "type": "long"
+ },
+ "subtype": {
+ "type": "string"
+ },
+ "schema": {
+ "dynamic": "true",
+ "properties": {
+ "version": {
+ "type": "integer"
+ }
+ }
+ },
+ "source": {
+ "type": "string"
+ },
+ "subtype": {
+ "type": "string"
+ },
+ "tags": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "title": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
new file mode 100644
index 0000000..c671b2e
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
@@ -0,0 +1,611 @@
+{
+ "type" : "object",
+ "$schema" : "http://json-schema.org/draft-03/schema",
+ "javaType" : "org.apache.streams.datasift.twitter.Twitter",
+ "properties" :
+ {
+ "created_at" :
+ {
+ "type" : "string"
+ },
+ "display_urls" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "domains" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "filter_level" :
+ {
+ "type" : "string"
+ },
+ "geo" :
+ {
+ "javaType" : "org.apache.streams.datasift.twitter.Geo",
+ "type" : "object",
+ "dynamic" : "true",
+ "properties" :
+ {
+ "latitude" :
+ {
+ "type" : "number"
+ },
+ "longitude" :
+ {
+ "type" : "number"
+ }
+ }
+ },
+ "hashtags" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "in_reply_to_screen_name" :
+ {
+ "type" : "string"
+ },
+ "in_reply_to_status_id" :
+ {
+ "type" : "string"
+ },
+ "in_reply_to_user_id" :
+ {
+ "type" : "string"
+ },
+ "lang" :
+ {
+ "type" : "string"
+ },
+ "links" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "media" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "javaType" : "org.apache.streams.datasift.twitter.Media",
+ "type" : "object",
+ "dynamic" : "true",
+ "properties" :
+ {
+ "display_url" :
+ {
+ "type" : "string"
+ },
+ "expanded_url" :
+ {
+ "type" : "string"
+ },
+ "id" :
+ {
+ "type" : "integer"
+ },
+ "id_str" :
+ {
+ "type" : "string"
+ },
+ "media_url" :
+ {
+ "type" : "string"
+ },
+ "media_url_https" :
+ {
+ "type" : "string"
+ },
+ "sizes" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "large" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ },
+ "medium" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ },
+ "small" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ },
+ "thumb" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ }
+ }
+ },
+ "source_status_id" :
+ {
+ "type" : "integer"
+ },
+ "source_status_id_str" :
+ {
+ "type" : "string"
+ },
+ "type" :
+ {
+ "type" : "string"
+ },
+ "url" :
+ {
+ "type" : "string"
+ }
+ }
+ }
+ ]
+ },
+ "mention_ids" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "mentions" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "place" :
+ {
+ "javaType" : "org.apache.streams.datasift.twitter.Place",
+ "type" : "object",
+ "dynamic" : "true",
+ "properties" :
+ {
+ "attributes" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "street_address" :
+ {
+ "type" : "string"
+ }
+ }
+ },
+ "country" :
+ {
+ "type" : "string"
+ },
+ "country_code" :
+ {
+ "type" : "string"
+ },
+ "full_name" :
+ {
+ "type" : "string"
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "name" :
+ {
+ "type" : "string"
+ },
+ "place_type" :
+ {
+ "type" : "string"
+ },
+ "url" :
+ {
+ "type" : "string"
+ }
+ }
+ },
+ "retweet" :
+ {
+ "javaType" : "org.apache.streams.datasift.twitter.Retweet",
+ "type" : "object",
+ "dynamic" : "true",
+ "properties" :
+ {
+ "count" :
+ {
+ "type" : "integer"
+ },
+ "created_at" :
+ {
+ "type" : "string"
+ },
+ "domains" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "hashtags" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "lang" :
+ {
+ "type" : "string"
+ },
+ "links" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "media" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "display_url" :
+ {
+ "type" : "string"
+ },
+ "expanded_url" :
+ {
+ "type" : "string"
+ },
+ "id" :
+ {
+ "type" : "integer"
+ },
+ "id_str" :
+ {
+ "type" : "string"
+ },
+ "media_url" :
+ {
+ "type" : "string"
+ },
+ "media_url_https" :
+ {
+ "type" : "string"
+ },
+ "sizes" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "large" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ },
+ "medium" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ },
+ "small" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ },
+ "thumb" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "h" :
+ {
+ "type" : "integer"
+ },
+ "resize" :
+ {
+ "type" : "string"
+ },
+ "w" :
+ {
+ "type" : "integer"
+ }
+ }
+ }
+ }
+ },
+ "source_status_id" :
+ {
+ "type" : "integer"
+ },
+ "source_status_id_str" :
+ {
+ "type" : "string"
+ },
+ "type" :
+ {
+ "type" : "string"
+ },
+ "url" :
+ {
+ "type" : "string"
+ }
+ }
+ },
+ "mention_ids" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "integer"
+ }
+ ]
+ },
+ "mentions" :
+ {
+ "type" : "array",
+ "items" :
+ [
+ {
+ "type" : "string"
+ }
+ ]
+ },
+ "source" :
+ {
+ "type" : "string"
+ },
+ "text" :
+ {
+ "type" : "string"
+ },
+ "user" :
+ {
+ "type": "object",
+ "$ref": "DatasiftTwitterUser.json"
+ }
+ }
+ },
+ "retweeted" :
+ {
+ "javaType" : "org.apache.streams.datasift.twitter.Retweeted",
+ "type" : "object",
+ "dynamic" : "true",
+ "properties" :
+ {
+ "created_at" :
+ {
+ "type" : "string"
+ },
+ "geo" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "latitude" :
+ {
+ "type" : "number"
+ },
+ "longitude" :
+ {
+ "type" : "number"
+ }
+ }
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "place" :
+ {
+ "dynamic" : "true",
+ "properties" :
+ {
+ "attributes" :
+ {
+ "type" : "object",
+ "dynamic" : "true"
+ },
+ "country" :
+ {
+ "type" : "string"
+ },
+ "country_code" :
+ {
+ "type" : "string"
+ },
+ "full_name" :
+ {
+ "type" : "string"
+ },
+ "id" :
+ {
+ "type" : "string"
+ },
+ "name" :
+ {
+ "type" : "string"
+ },
+ "place_type" :
+ {
+ "type" : "string"
+ },
+ "url" :
+ {
+ "type" : "string"
+ }
+ }
+ },
+ "source" :
+ {
+ "type" : "string"
+ },
+ "user" :
+ {
+ "type": "object",
+ "$ref": "DatasiftTwitterUser.json"
+ }
+ }
+ },
+ "source" :
+ {
+ "type" : "string"
+ },
+ "text" :
+ {
+ "type" : "string"
+ },
+ "user" :
+ {
+ "type": "object",
+ "$ref": "DatasiftTwitterUser.json"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterUser.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterUser.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterUser.json
new file mode 100644
index 0000000..3be49ff
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterUser.json
@@ -0,0 +1,68 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "javaType": "org.apache.streams.datasift.twitter.DatasiftTwitterUser",
+ "properties": {
+ "created_at": {
+ "type": "string",
+ "format": "date-time"
+ },
+ "description": {
+ "type": "string"
+ },
+ "favourites_count": {
+ "type": "integer"
+ },
+ "followers_count": {
+ "type": "integer"
+ },
+ "friends_count": {
+ "type": "integer"
+ },
+ "geo_enabled": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "listed_count": {
+ "type": "integer"
+ },
+ "location": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "profile_image_url": {
+ "type": "string"
+ },
+ "profile_image_url_https": {
+ "type": "string"
+ },
+ "screen_name": {
+ "type": "string"
+ },
+ "statuses_count": {
+ "type": "integer"
+ },
+ "time_zone": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ },
+ "utc_offset": {
+ "type": "integer"
+ },
+ "verified": {
+ "type": "boolean"
+ }
+ }
+}
\ No newline at end of file
[02/15] git commit: optimized imports in pogo to resolve local class
path issues
Posted by sb...@apache.org.
optimized imports in pogo to resolve local class path issues
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/be5a936e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/be5a936e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/be5a936e
Branch: refs/heads/master
Commit: be5a936e21586f28e370d58748de669e5aba059b
Parents: 0cf06a5
Author: sblackmon <sb...@w2odigital.com>
Authored: Sun Aug 10 12:18:21 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../main/java/org/apache/streams/data/util/JsonUtil.java | 8 ++++++--
.../apache/streams/jackson/StreamsDateTimeSerializer.java | 4 ----
.../org/apache/streams/jackson/StreamsJacksonMapper.java | 10 ----------
.../apache/streams/jackson/StreamsPeriodDeserializer.java | 1 -
.../apache/streams/jackson/StreamsPeriodSerializer.java | 2 --
.../apache/streams/data/data/util/DateTimeSerDeTest.java | 2 --
6 files changed, 6 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/be5a936e/streams-pojo/src/main/java/org/apache/streams/data/util/JsonUtil.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/data/util/JsonUtil.java b/streams-pojo/src/main/java/org/apache/streams/data/util/JsonUtil.java
index 79ac555..6ab2800 100644
--- a/streams-pojo/src/main/java/org/apache/streams/data/util/JsonUtil.java
+++ b/streams-pojo/src/main/java/org/apache/streams/data/util/JsonUtil.java
@@ -31,8 +31,12 @@ import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import org.apache.streams.jackson.StreamsJacksonMapper;
-import java.io.*;
-import java.util.*;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
/**
* JSON utilities
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/be5a936e/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
index ba8a3e0..1e9c895 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
@@ -18,15 +18,11 @@
package org.apache.streams.jackson;
-import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.apache.streams.data.util.RFC3339Utils;
import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
import java.io.IOException;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/be5a936e/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
index 40dd244..25c0c89 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
@@ -21,19 +21,9 @@ package org.apache.streams.jackson;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-
-import java.io.IOException;
/**
* Created by sblackmon on 3/27/14.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/be5a936e/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
index 7a98229..56487cd 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
@@ -21,7 +21,6 @@ package org.apache.streams.jackson;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import org.apache.streams.data.util.RFC3339Utils;
import org.joda.time.Period;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/be5a936e/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
index 5121ee9..f72d337 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
@@ -21,8 +21,6 @@ package org.apache.streams.jackson;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-import org.apache.streams.data.util.RFC3339Utils;
-import org.joda.time.DateTime;
import org.joda.time.Period;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/be5a936e/streams-pojo/src/test/java/org/apache/streams/data/data/util/DateTimeSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/test/java/org/apache/streams/data/data/util/DateTimeSerDeTest.java b/streams-pojo/src/test/java/org/apache/streams/data/data/util/DateTimeSerDeTest.java
index 2cb02b3..b9c5635 100644
--- a/streams-pojo/src/test/java/org/apache/streams/data/data/util/DateTimeSerDeTest.java
+++ b/streams-pojo/src/test/java/org/apache/streams/data/data/util/DateTimeSerDeTest.java
@@ -27,8 +27,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
import static org.junit.Assert.assertEquals;
/**
[14/15] git commit: moved json cleaner into streams-processor-jackson
better javadoc comment
Posted by sb...@apache.org.
moved json cleaner into streams-processor-jackson
better javadoc comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/0536c98a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/0536c98a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/0536c98a
Branch: refs/heads/master
Commit: 0536c98a845def2e09c7fa14ca8c0c4df0666505
Parents: 614f403
Author: sblackmon <sb...@apache.org>
Authored: Thu Aug 21 16:49:35 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Thu Aug 21 16:49:35 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 1 +
.../streams-processor-jackson/pom.xml | 86 ++++++++++++++++++++
.../CleanAdditionalPropertiesProcessor.java | 62 ++++++++++++++
.../streams-provider-datasift/pom.xml | 5 ++
.../CleanAdditionalPropertiesProcessor.java | 62 --------------
.../DatasiftTypeConverterProcessor.java | 1 +
6 files changed, 155 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
index 699274e..e290466 100644
--- a/streams-contrib/pom.xml
+++ b/streams-contrib/pom.xml
@@ -48,6 +48,7 @@
<!--<module>streams-processor-lucene</module>-->
<!--<module>streams-processor-tika</module>-->
<module>streams-provider-instagram</module>
+ <module>streams-processor-jackson</module>
<module>streams-processor-json</module>
<module>streams-processor-urls</module>
<module>streams-provider-datasift</module>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-processor-jackson/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-jackson/pom.xml b/streams-contrib/streams-processor-jackson/pom.xml
new file mode 100644
index 0000000..8bb44fd
--- /dev/null
+++ b/streams-contrib/streams-processor-jackson/pom.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>streams-processor-jackson</artifactId>
+ <version>0.1-SNAPSHOT</version>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-contrib</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.jayway.jsonpath</groupId>
+ <artifactId>json-path</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.jayway.jsonpath</groupId>
+ <artifactId>json-path-assert</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java b/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
new file mode 100644
index 0000000..56b0c5c
--- /dev/null
+++ b/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
@@ -0,0 +1,62 @@
+package org.apache.streams.jackson;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
+import com.google.common.collect.Lists;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This processor walks an input objectnode and corrects any artifacts
+ * that may have occured from improper serialization of jsonschema2pojo beans.
+ *
+ * The logic is also available for inclusion in other module via static import.
+ */
+public class CleanAdditionalPropertiesProcessor implements StreamsProcessor {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(CleanAdditionalPropertiesProcessor.class);
+
+ private ObjectMapper mapper;
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum datum) {
+ List<StreamsDatum> result = Lists.newLinkedList();
+ ObjectNode activity = this.mapper.convertValue(datum.getDocument(), ObjectNode.class);
+ cleanAdditionalProperties(activity);
+ datum.setDocument(activity);
+ result.add(datum);
+ return result;
+ }
+
+ @Override
+ public void prepare(Object o) {
+ this.mapper = StreamsJacksonMapper.getInstance();
+ this.mapper.registerModule(new JsonOrgModule());
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ public static void cleanAdditionalProperties(ObjectNode node) {
+ if( node.get("additionalProperties") != null ) {
+ ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
+ cleanAdditionalProperties(additionalProperties);
+ Iterator<Map.Entry<String, JsonNode>> jsonNodeIterator = additionalProperties.fields();
+ while( jsonNodeIterator.hasNext() ) {
+ Map.Entry<String, JsonNode> entry = jsonNodeIterator.next();
+ node.put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-provider-datasift/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml
index f2aab08..782db07 100644
--- a/streams-contrib/streams-provider-datasift/pom.xml
+++ b/streams-contrib/streams-provider-datasift/pom.xml
@@ -37,6 +37,11 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
+ <artifactId>streams-processor-jackson</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
<artifactId>streams-provider-twitter</artifactId>
<version>${project.version}</version>
<optional>true</optional>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
deleted file mode 100644
index 00f08e7..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.apache.streams.datasift.processor;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
-import com.google.common.collect.Lists;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * HACK PROCESSOR. Changes need to be made in apache streams to fix this issue long term.
- */
-public class CleanAdditionalPropertiesProcessor implements StreamsProcessor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(CleanAdditionalPropertiesProcessor.class);
-
- private static final String ADDITIONAL_PROPERTIES = "additionalProperties";
- private static final String EXTENSIONS = "extensions";
-
- private ObjectMapper mapper;
-
- @Override
- public List<StreamsDatum> process(StreamsDatum datum) {
- List<StreamsDatum> result = Lists.newLinkedList();
- ObjectNode activity = this.mapper.convertValue(datum.getDocument(), ObjectNode.class);
- cleanAdditionalProperties(activity);
- datum.setDocument(activity);
- result.add(datum);
- return result;
- }
-
- @Override
- public void prepare(Object o) {
- this.mapper = StreamsJacksonMapper.getInstance();
- this.mapper.registerModule(new JsonOrgModule());
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- public static void cleanAdditionalProperties(ObjectNode node) {
- if( node.get("additionalProperties") != null ) {
- ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
- cleanAdditionalProperties(additionalProperties);
- Iterator<Map.Entry<String, JsonNode>> jsonNodeIterator = additionalProperties.fields();
- while( jsonNodeIterator.hasNext() ) {
- Map.Entry<String, JsonNode> entry = jsonNodeIterator.next();
- node.put(entry.getKey(), entry.getValue());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
index e85e8b1..a00cf23 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
@@ -28,6 +28,7 @@ import org.apache.streams.datasift.provider.DatasiftConverter;
import org.apache.streams.datasift.serializer.DatasiftActivitySerializer;
import org.apache.streams.datasift.util.StreamsDatasiftMapper;
import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.jackson.CleanAdditionalPropertiesProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
[08/15] git commit: easier to use existing jackson-annotated beans
Posted by sb...@apache.org.
easier to use existing jackson-annotated beans
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e6380ceb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e6380ceb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e6380ceb
Branch: refs/heads/master
Commit: e6380cebed7f4ce8d211cc7e2e73d0080041cc71
Parents: 6381dc9
Author: sblackmon <sb...@w2odigital.com>
Authored: Fri Aug 8 14:50:27 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../provider/DatasiftManagedSourceSetup.java | 69 ++++++++++++++
.../provider/DatasiftStreamConfigurator.java | 10 ++-
.../provider/DatasiftStreamProvider.java | 2 +-
.../com/datasift/DatasiftConfiguration.json | 94 +-------------------
4 files changed, 79 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e6380ceb/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
new file mode 100644
index 0000000..6e648e7
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
@@ -0,0 +1,69 @@
+package org.apache.streams.datasift.provider;
+
+import com.datasift.client.DataSiftClient;
+import com.datasift.client.FutureData;
+import com.datasift.client.managedsource.ManagedSource;
+import com.datasift.client.managedsource.ManagedSourceList;
+import com.datasift.client.managedsource.sources.DataSource;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.streams.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.datasift.DatasiftConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Created by sblackmon on 8/8/14.
+ */
+public class DatasiftManagedSourceSetup implements Runnable {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamProvider.class);
+
+ private static DatasiftConfiguration config = DatasiftStreamConfigurator.detectConfiguration(StreamsConfigurator.config);
+
+ DataSiftClient client;
+ Map<String, ManagedSource> currentManagedSourceMap = Maps.newHashMap();
+ List<ManagedSource> updatedManagedSourceList;
+
+ public static void main(String[] args) {
+ DatasiftManagedSourceSetup job = new DatasiftManagedSourceSetup();
+ (new Thread(job)).start();
+ }
+
+ @Override
+ public void run() {
+
+ setup();
+
+ current();
+
+ updatedManagedSourceList = config.getManagedSources();
+
+ for( ManagedSource source : updatedManagedSourceList ) {
+ ManagedSource current = currentManagedSourceMap.get( source.getId() );
+ ManagedSource updated = client.managedSource().update(current.getName(), (DataSource) source, current).sync();
+ }
+
+ }
+
+ public void setup() {
+ client = DatasiftStreamProvider.getNewClient(config.getUserName(), config.getApiKey());
+ }
+
+ public void current() {
+ ManagedSourceList managedSources = client.managedSource().get().sync();
+ Iterator<ManagedSource> managedSourceIterator = managedSources.iterator();
+ while( managedSourceIterator.hasNext() ) {
+ ManagedSource source = managedSourceIterator.next();
+ currentManagedSourceMap.put(source.getId(), source);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e6380ceb/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
index 7025d39..5726b4f 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
@@ -34,8 +34,14 @@ public class DatasiftStreamConfigurator {
public static DatasiftConfiguration detectConfiguration(Config datasift) {
- DatasiftConfiguration datasiftConfiguration;
- datasiftConfiguration = StreamsJacksonMapper.getInstance().convertValue(datasift.root().render(ConfigRenderOptions.concise()), DatasiftConfiguration.class);
+ DatasiftConfiguration datasiftConfiguration = null;
+
+ try {
+ datasiftConfiguration = mapper.readValue(datasift.root().render(ConfigRenderOptions.concise()), DatasiftConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse datasiftConfiguration");
+ }
return datasiftConfiguration;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e6380ceb/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
index 8ed1443..c3f5ea3 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
@@ -128,7 +128,7 @@ public class DatasiftStreamProvider implements StreamsProvider {
* @param apiKey
* @return
*/
- protected DataSiftClient getNewClient(String userName, String apiKey) {
+ protected static DataSiftClient getNewClient(String userName, String apiKey) {
return new DataSiftClient(new DataSiftConfig(userName, apiKey));
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e6380ceb/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
index ae9daa6..e024f43 100644
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
@@ -31,100 +31,8 @@
"type": "array",
"minItems": 0,
"items": {
- "javaType": "org.apache.streams.datasift.managed.ManagedSource",
"type": "object",
- "properties": {
- "auth": {
- "type": "array",
- "items": {
- "javaType": "org.apache.streams.datasift.managed.ManagedSourceAuth",
- "type": "object",
- "properties": {
- "identity_id": {
- "type": "string"
- },
- "parameters": {
- "type": "object",
- "properties": {
- "value": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "created_at": {
- "type": "number"
- },
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "parameters": {
- "javaType": "org.apache.streams.datasift.managed.ManagedSourceParameters",
- "type": "object",
- "properties": {
- "comments": {
- "type": "boolean"
- },
- "likes": {
- "type": "boolean"
- }
- }
- },
- "resources": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.managed.ManagedSourceResource",
- "type": "object",
- "properties": {
- "parameters": {
- "type": "object",
- "properties": {
- "distance": {
- "type": "string"
- },
- "exact_match": {
- "type": "string"
- },
- "foursq": {
- "type": "string"
- },
- "lat": {
- "type": "string"
- },
- "lng": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "usermeta": {
- "type": "string"
- },
- "value": {
- "type": "number"
- }
- }
- },
- "resource_id": {
- "type": "string"
- }
- }
- }
- ]
- },
- "source_type": {
- "type": "string"
- },
- "status": {
- "type": "string"
- }
- }
+ "javaType": "com.datasift.client.managedsource.ManagedSource"
}
}
}
[05/15] git commit: channel serializers call super on default
serializer first hashtags working still problems with additionalProperties
Posted by sb...@apache.org.
channel serializers call super on default serializer first
hashtags working
still problems with additionalProperties
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d387b1d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d387b1d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d387b1d4
Branch: refs/heads/master
Commit: d387b1d42e41c54e95462a2c093ea9564a6df0b9
Parents: e8c85ee
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu Aug 7 17:05:43 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:23 2014 -0500
----------------------------------------------------------------------
.../streams-provider-datasift/pom.xml | 9 +-
.../serializer/DatasiftActivitySerializer.java | 5 +-
.../DatasiftDefaultActivitySerializer.java | 11 +-
.../DatasiftInstagramActivitySerializer.java | 124 +++++++++++++++++++
.../DatasiftTweetActivitySerializer.java | 6 +
.../main/jsonschema/com/datasift/Datasift.json | 96 +++++++++++---
.../DatasiftActivitySerializerTest.java | 4 +-
7 files changed, 232 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml
index 3b9f96f..b68e261 100644
--- a/streams-contrib/streams-provider-datasift/pom.xml
+++ b/streams-contrib/streams-provider-datasift/pom.xml
@@ -39,11 +39,18 @@
<groupId>org.apache.streams</groupId>
<artifactId>streams-provider-twitter</artifactId>
<version>${project.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-provider-instagram</artifactId>
+ <version>${project.version}</version>
+ <optional>true</optional>
</dependency>
<dependency>
<groupId>com.datasift.client</groupId>
<artifactId>datasift-java</artifactId>
- <version>3.0.0-Beta3.4</version>
+ <version>3.0.0-Beta3.5</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
index 7d644f0..1e4f63c 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
@@ -32,8 +32,9 @@ import java.util.List;
*/
public class DatasiftActivitySerializer implements ActivitySerializer<Datasift> {
- private static final DatasiftTweetActivitySerializer TWITTER_SERIALIZER = new DatasiftTweetActivitySerializer();
private static final DatasiftDefaultActivitySerializer DEFAULT_SERIALIZER = new DatasiftDefaultActivitySerializer();
+ private static final DatasiftTweetActivitySerializer TWITTER_SERIALIZER = new DatasiftTweetActivitySerializer();
+ private static final DatasiftInstagramActivitySerializer INSTAGRAM_SERIALIZER = new DatasiftInstagramActivitySerializer();
private static final ObjectMapper MAPPER = StreamsDatasiftMapper.getInstance();
@Override
@@ -50,6 +51,8 @@ public class DatasiftActivitySerializer implements ActivitySerializer<Datasift>
public Activity deserialize(Datasift serialized) throws ActivitySerializerException {
if(serialized.getTwitter() != null) {
return TWITTER_SERIALIZER.deserialize(serialized);
+ } else if(serialized.getInstagram() != null) {
+ return INSTAGRAM_SERIALIZER.deserialize(serialized);
} else {
return DEFAULT_SERIALIZER.deserialize(serialized);
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
index b70aa12..615ca34 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftDefaultActivitySerializer.java
@@ -109,7 +109,7 @@ public class DatasiftDefaultActivitySerializer implements ActivitySerializer<Dat
Activity activity = new Activity();
activity.setActor(buildActor(event.getInteraction()));
- activity.setVerb("post");
+ activity.setVerb(selectVerb(event));
activity.setObject(buildActivityObject(event.getInteraction()));
activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
activity.setTarget(buildTarget(event.getInteraction()));
@@ -128,6 +128,10 @@ public class DatasiftDefaultActivitySerializer implements ActivitySerializer<Dat
return activity;
}
+ private String selectVerb(Datasift event) {
+ return "post";
+ }
+
public Actor buildActor(Interaction interaction) {
Actor actor = new Actor();
org.apache.streams.datasift.interaction.Author author = interaction.getAuthor();
@@ -202,4 +206,9 @@ public class DatasiftDefaultActivitySerializer implements ActivitySerializer<Dat
extensions.put("location", location);
}
+ public static String firstStringIfNotNull(List<Object> list) {
+ if( list != null && list.size() > 0) {
+ return (String) list.get(0);
+ } else return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
new file mode 100644
index 0000000..54f14af
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
@@ -0,0 +1,124 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+package org.apache.streams.datasift.serializer;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.streams.data.util.RFC3339Utils;
+import org.apache.streams.datasift.Datasift;
+import org.apache.streams.datasift.instagram.From;
+import org.apache.streams.datasift.instagram.Instagram;
+import org.apache.streams.datasift.interaction.Interaction;
+import org.apache.streams.instagram.serializer.util.InstagramActivityUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.apache.streams.pojo.json.Image;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
+
+/**
+ *
+ */
+public class DatasiftInstagramActivitySerializer extends DatasiftDefaultActivitySerializer {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftInstagramActivitySerializer.class);
+
+ @Override
+ public Activity convert(Datasift event) {
+ Activity activity = super.convert(event);
+
+ Instagram instagram = event.getInstagram();
+
+ activity.setActor(buildActor(event, instagram));
+ activity.setObject(buildActivityObject(event.getInteraction()));
+ activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
+ activity.setTarget(buildTarget(event.getInteraction()));
+ activity.setPublished(event.getInteraction().getCreatedAt());
+ activity.setGenerator(buildGenerator(event.getInteraction()));
+ activity.setIcon(getIcon(event.getInteraction()));
+ activity.setProvider(InstagramActivityUtil.getProvider());
+ activity.setTitle(event.getInteraction().getTitle());
+ activity.setUrl(event.getInteraction().getLink());
+
+ activity.setVerb(selectVerb(event));
+ addInstagramExtensions(activity, instagram);
+ return activity;
+ }
+
+ /**
+ * Gets links from the object
+ * @return
+ */
+ private List<String> getLinks(Instagram instagram) {
+ List<String> result = Lists.newLinkedList();
+ result.add(instagram.getMedia().getImage());
+ result.add(instagram.getMedia().getVideo());
+ return result;
+ }
+
+ public Actor buildActor(Datasift event, Instagram instagram) {
+ Actor actor = super.buildActor(event.getInteraction());
+ From user = instagram.getFrom();
+
+ actor.setDisplayName(firstStringIfNotNull(user.getFullName()));
+ actor.setId(formatId(Optional.fromNullable(
+ user.getId())
+ .or(actor.getId())));
+
+ Image profileImage = new Image();
+ String profileUrl = null;
+ profileUrl = user.getProfilePicture();
+ profileImage.setUrl(profileUrl);
+ actor.setImage(profileImage);
+
+ return actor;
+ }
+
+ public void addInstagramExtensions(Activity activity, Instagram instagram) {
+ Map<String, Object> extensions = ensureExtensions(activity);
+ List<String> hashTags;
+ if(instagram.getMedia() != null) {
+ hashTags = instagram.getMedia().getTags();
+ extensions.put("hashtags", hashTags);
+ extensions.put("keywords", instagram.getMedia().getCaption());
+ }
+ }
+
+ private String selectVerb(Datasift event) {
+ if( event.getInteraction().getSubtype().equals("like"))
+ return "like";
+ else
+ return "post";
+ }
+
+ public static String formatId(String... idparts) {
+ return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
index b16aae2..3c7abda 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
@@ -19,6 +19,7 @@ under the License.
package org.apache.streams.datasift.serializer;
+import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -257,4 +258,9 @@ public class DatasiftTweetActivitySerializer extends DatasiftDefaultActivitySeri
return userMentions;
}
+
+ public static String formatId(String... idparts) {
+ return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
index 37a63a0..fdf1aac 100644
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
@@ -150,6 +150,77 @@
}
}
},
+ "instagram": {
+ "javaType": "org.apache.streams.datasift.instagram.Instagram",
+ "type": "object",
+ "properties": {
+ "from": {
+ "javaType": "org.apache.streams.datasift.instagram.From",
+ "type": "object",
+ "properties": {
+ "full_name": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "id": {
+ "type": "string"
+ },
+ "profile_picture": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "media": {
+ "javaType": "org.apache.streams.datasift.instagram.Media",
+ "type": "object",
+ "properties": {
+ "caption": {
+ "type": "string"
+ },
+ "created_time": {
+ "type": "number"
+ },
+ "filter": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "image": {
+ "type": "string"
+ },
+ "link": {
+ "type": "string"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "type": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ },
+ "video": {
+ "type": "string"
+ }
+ }
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ },
"interaction": {
"javaType": "org.apache.streams.datasift.interaction.Interaction",
"type": "object",
@@ -206,23 +277,10 @@
"link": {
"type": "string"
},
- "mention_ids": {
- "type": "array",
- "items": [
- {
- "type": "integer"
- }
- ]
- },
- "mentions": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
+ "received_at": {
+ "type": "long"
},
- "source": {
+ "subtype": {
"type": "string"
},
"schema": {
@@ -236,6 +294,9 @@
"source": {
"type": "string"
},
+ "subtype": {
+ "type": "string"
+ },
"tags": {
"type": "array",
"items": [
@@ -955,12 +1016,11 @@
},
"mention_ids": {
"type": "array",
- "items":[
+ "items": [
{
"type": "integer"
}
]
-
},
"mentions": {
"type": "array",
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d387b1d4/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
index daff2d7..baff556 100644
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
@@ -62,7 +62,6 @@ public class DatasiftActivitySerializerTest {
}
}
-
/**
* Test that the minimum number of things that an activity has
* @param json
@@ -70,9 +69,10 @@ public class DatasiftActivitySerializerTest {
private void testGeneralConversion(String json) throws Exception {
Activity activity = SERIALIZER.deserialize(json);
assertNotNull(json, activity.getId());
- assertTrue(json, activity.getId().contains("datasift"));
assertNotNull(json, activity.getPublished());
+ assertNotNull(json, activity.getProvider());
assertNotNull(json, activity.getUrl());
+ assertNotNull(json, activity.getVerb());
Actor actor = activity.getActor();
assertNotNull(json, actor);
[13/15] git commit: broke channels within Datasift.json out into
fragments
Posted by sb...@apache.org.
broke channels within Datasift.json out into fragments
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/199881d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/199881d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/199881d5
Branch: refs/heads/master
Commit: 199881d547c996fdbde3c07ddb6a28c417d365da
Parents: 4cb20a1
Author: sblackmon <sb...@apache.org>
Authored: Mon Aug 11 18:24:58 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Aug 12 18:04:24 2014 -0500
----------------------------------------------------------------------
pom.xml | 2 +-
.../streams-provider-datasift/pom.xml | 7 +-
.../streams/datasift/csdl/DatasiftCsdlUtil.java | 6 +-
.../provider/DatasiftManagedSourceSetup.java | 5 -
.../provider/DatasiftStreamConfigurator.java | 1 -
.../DatasiftInstagramActivitySerializer.java | 6 -
.../main/jsonschema/com/datasift/Datasift.json | 1226 ------------------
.../com/datasift/DatasiftConfiguration.json | 133 --
.../com/datasift/DatasiftPushConfiguration.json | 17 -
.../datasift/DatasiftStreamConfiguration.json | 17 -
.../com/datasift/DatasiftTwitterUser.json | 68 -
.../org/apache/streams/datasift/Datasift.json | 458 +++++++
.../streams/datasift/DatasiftConfiguration.json | 133 ++
.../datasift/DatasiftPushConfiguration.json | 17 +
.../datasift/DatasiftStreamConfiguration.json | 17 +
.../datasift/facebook/DatasiftFacebook.json | 120 ++
.../datasift/instagram/DatasiftInstagram.json | 178 +++
.../interaction/DatasiftInteraction.json | 92 ++
.../datasift/twitter/DatasiftTwitter.json | 611 +++++++++
.../datasift/twitter/DatasiftTwitterUser.json | 68 +
20 files changed, 1700 insertions(+), 1482 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 419ef89..5f5d5d2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -69,7 +69,7 @@
<joda-time.version>2.2</joda-time.version>
<rave.version>0.22</rave.version>
<datastax.version>1.0.3</datastax.version>
- <jsonschema2pojo.version>0.4.1</jsonschema2pojo.version>
+ <jsonschema2pojo.version>0.4.5</jsonschema2pojo.version>
<jaxb2.version>0.8.3</jaxb2.version>
<jaxb2-basics.version>0.6.5</jaxb2-basics.version>
<jaxbutil.version>1.2.6</jaxbutil.version>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml
index b68e261..f2aab08 100644
--- a/streams-contrib/streams-provider-datasift/pom.xml
+++ b/streams-contrib/streams-provider-datasift/pom.xml
@@ -127,12 +127,9 @@
<configuration>
<addCompileSourceRoot>true</addCompileSourceRoot>
<generateBuilders>true</generateBuilders>
- <sourcePaths>
- <sourcePath>src/main/jsonschema/com/datasift/Datasift.json</sourcePath>
- <sourcePath>src/main/jsonschema/com/datasift/DatasiftConfiguration.json</sourcePath>
- </sourcePaths>
+ <sourceDirectory>src/main/jsonschema</sourceDirectory>
<outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>com.datasift</targetPackage>
+ <targetPackage>org.apache.streams.datasift</targetPackage>
<useLongIntegers>true</useLongIntegers>
<useJodaDates>true</useJodaDates>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/csdl/DatasiftCsdlUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/csdl/DatasiftCsdlUtil.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/csdl/DatasiftCsdlUtil.java
index e317e51..7ff03fb 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/csdl/DatasiftCsdlUtil.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/csdl/DatasiftCsdlUtil.java
@@ -1,12 +1,12 @@
package org.apache.streams.datasift.csdl;
-import java.util.List;
-import java.util.ListIterator;
-
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.List;
+import java.util.ListIterator;
+
public class DatasiftCsdlUtil {
private static final Logger log = LoggerFactory
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
index 9e2f99e..4939180 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
@@ -1,15 +1,11 @@
package org.apache.streams.datasift.provider;
import com.datasift.client.DataSiftClient;
-import com.datasift.client.FutureData;
import com.datasift.client.managedsource.ManagedSource;
import com.datasift.client.managedsource.ManagedSourceList;
import com.datasift.client.managedsource.sources.DataSource;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.streams.StreamsConfiguration;
import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.datasift.DatasiftConfiguration;
import org.apache.streams.datasift.managed.StreamsManagedSource;
@@ -20,7 +16,6 @@ import org.slf4j.LoggerFactory;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
/**
* Created by sblackmon on 8/8/14.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
index 07716a1..b9d31ff 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
@@ -23,7 +23,6 @@ import com.typesafe.config.Config;
import com.typesafe.config.ConfigRenderOptions;
import org.apache.streams.datasift.DatasiftConfiguration;
import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.jackson.StreamsJacksonMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
index 4149fe3..cb44df2 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
@@ -21,22 +21,16 @@ package org.apache.streams.datasift.serializer;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.streams.data.util.RFC3339Utils;
import org.apache.streams.datasift.Datasift;
import org.apache.streams.datasift.instagram.From;
import org.apache.streams.datasift.instagram.Instagram;
-import org.apache.streams.datasift.interaction.Interaction;
import org.apache.streams.instagram.serializer.util.InstagramActivityUtil;
import org.apache.streams.pojo.json.Activity;
import org.apache.streams.pojo.json.Actor;
import org.apache.streams.pojo.json.Image;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
deleted file mode 100644
index 0c21302..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
+++ /dev/null
@@ -1,1226 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "javaType": "org.apache.streams.datasift.Datasift",
- "properties": {
- "blog": {
- "javaType": "org.apache.streams.datasift.blog.Blog",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "type": "object",
- "javaType": "org.apache.streams.datasift.blog.BlogAuthor",
- "dynamic": "true",
- "properties": {
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "domain": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- },
- "demographic": {
- "dynamic": "true",
- "properties": {
- "gender": {
- "type": "string"
- }
- }
- },
- "facebook": {
- "javaType": "org.apache.streams.datasift.config.Facebook",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "application": {
- "type": "string"
- },
- "author": {
- "javaType": "org.apache.streams.datasift.config.Author",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "avatar": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "id": {
- "type": "string"
- },
- "likes": {
- "javaType": "org.apache.streams.datasift.config.Likes",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "count": {
- "type": "integer"
- },
- "ids": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "names": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "message": {
- "type": "string"
- },
- "source": {
- "type": "string"
- },
- "to": {
- "javaType": "org.apache.streams.datasift.config.To",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "ids": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "names": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "type": {
- "type": "string"
- }
- }
- },
- "instagram": {
- "javaType": "org.apache.streams.datasift.instagram.Instagram",
- "type": "object",
- "properties": {
- "caption": {
- "type": "object",
- "properties": {
- "created_time": {
- "type": "number"
- },
- "id": {
- "type": "string"
- },
- "text": {
- "type": "string"
- },
- "from": {
- "type": "object",
- "properties": {
- "full_name": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "id": {
- "type": "string"
- },
- "profile_picture": {
- "type": "string"
- },
- "username": {
- "type": "string"
- }
- }
- }
- }
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "from": {
- "javaType": "org.apache.streams.datasift.instagram.From",
- "type": "object",
- "properties": {
- "full_name": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "id": {
- "type": "string"
- },
- "profile_picture": {
- "type": "string"
- },
- "username": {
- "type": "string"
- }
- }
- },
- "id": {
- "type": "string"
- },
- "images": {
- "type": "object",
- "properties": {
- "low_resolution": {
- "type": "object"
- },
- "standard_resolution": {
- "type": "object"
- },
- "thumbnail": {
- "type": "object"
- }
- }
- },
- "media": {
- "javaType": "org.apache.streams.datasift.instagram.Media",
- "type": "object",
- "properties": {
- "caption": {
- "type": "string"
- },
- "created_time": {
- "type": "number"
- },
- "filter": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "tags": {
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "type": {
- "type": "string"
- },
- "username": {
- "type": "string"
- },
- "video": {
- "type": "string"
- }
- }
- },
- "tags": {
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "type": {
- "type": "string"
- }
- }
- },
- "interaction": {
- "javaType": "org.apache.streams.datasift.interaction.Interaction",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "javaType": "org.apache.streams.datasift.interaction.Author",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "avatar": {
- "type": "string"
- },
- "id": {
- "type": "integer"
- },
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "username": {
- "type": "string"
- }
- }
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "geo": {
- "javaType": "org.apache.streams.datasift.interaction.Geo",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "latitude": {
- "type": "number"
- },
- "longitude": {
- "type": "number"
- }
- }
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "received_at": {
- "type": "long"
- },
- "subtype": {
- "type": "string"
- },
- "schema": {
- "dynamic": "true",
- "properties": {
- "version": {
- "type": "integer"
- }
- }
- },
- "source": {
- "type": "string"
- },
- "subtype": {
- "type": "string"
- },
- "tags": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- },
- "klout": {
- "javaType": "org.apache.streams.datasift.klout.Klout",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "score": {
- "type": "integer"
- }
- }
- },
- "language": {
- "javaType": "org.apache.streams.datasift.interaction.Language",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "confidence": {
- "type": "integer"
- },
- "tag": {
- "type": "string"
- },
- "tag_extended": {
- "type": "string"
- }
- }
- },
- "links": {
- "javaType": "org.apache.streams.datasift.interaction.Links",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "code": {
- "type": "array",
- "items": [
- {
- "type": "integer"
- }
- ]
- },
- "created_at": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "hops": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "meta": {
- "dynamic": "true",
- "javaType": "org.apache.streams.datasift.interaction.Meta",
- "type": "object",
- "properties": {
- "charset": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "content_type": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "description": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "keywords": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "lang": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "opengraph": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.interaction.meta.OpenGraph",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "description": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "site_name": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- },
- "twitter": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.interaction.meta.Twitter",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "card": {
- "type": "string"
- },
- "creator": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "player": {
- "type": "string"
- },
- "player_height": {
- "type": "string"
- },
- "player_width": {
- "type": "string"
- },
- "site": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- }
- }
- },
- "normalized_url": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "retweet_count": {
- "type": "array",
- "items": [
- {
- "type": "long"
- }
- ]
- },
- "title": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "url": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "salience": {
- "javaType": "org.apache.streams.datasift.salience.Salience",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "content": {
- "javaType": "org.apache.streams.datasift.salience.content.Content",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "entities": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.salience.content.Entities",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "about": {
- "type": "integer"
- },
- "confident": {
- "type": "integer"
- },
- "evidence": {
- "type": "integer"
- },
- "label": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "sentiment": {
- "type": "integer"
- },
- "themes": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- }
- ]
- },
- "sentiment": {
- "type": "integer"
- }
- }
- },
- "title": {
- "javaType": "org.apache.streams.datasift.salience.title.Title",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "entities": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.salience.title.Entities",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "about": {
- "type": "integer"
- },
- "confident": {
- "type": "integer"
- },
- "evidence": {
- "type": "integer"
- },
- "label": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "sentiment": {
- "type": "integer"
- },
- "themes": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- }
- ]
- },
- "sentiment": {
- "type": "integer"
- }
- }
- }
- }
- },
- "trends": {
- "javaType": "org.apache.streams.datasift.trends.Trends",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "content": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "source": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "type": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "twitter": {
- "javaType": "org.apache.streams.datasift.twitter.Twitter",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "created_at": {
- "type": "string"
- },
- "display_urls": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "domains": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "filter_level": {
- "type": "string"
- },
- "geo": {
- "javaType": "org.apache.streams.datasift.twitter.Geo",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "latitude": {
- "type": "number"
- },
- "longitude": {
- "type": "number"
- }
- }
- },
- "hashtags": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "id": {
- "type": "string"
- },
- "in_reply_to_screen_name": {
- "type": "string"
- },
- "in_reply_to_status_id": {
- "type": "string"
- },
- "in_reply_to_user_id": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "links": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "media": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.twitter.Media",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "display_url": {
- "type": "string"
- },
- "expanded_url": {
- "type": "string"
- },
- "id": {
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "media_url": {
- "type": "string"
- },
- "media_url_https": {
- "type": "string"
- },
- "sizes": {
- "dynamic": "true",
- "properties": {
- "large": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "integer"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "integer"
- }
- }
- },
- "medium": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "integer"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "integer"
- }
- }
- },
- "small": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "integer"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "integer"
- }
- }
- },
- "thumb": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "integer"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "integer"
- }
- }
- }
- }
- },
- "source_status_id": {
- "type": "integer"
- },
- "source_status_id_str": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- },
- "mention_ids": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "mentions": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "place": {
- "javaType": "org.apache.streams.datasift.twitter.Place",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "attributes": {
- "dynamic": "true",
- "properties": {
- "street_address": {
- "type": "string"
- }
- }
- },
- "country": {
- "type": "string"
- },
- "country_code": {
- "type": "string"
- },
- "full_name": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "place_type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "retweet": {
- "javaType": "org.apache.streams.datasift.twitter.Retweet",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "count": {
- "type": "integer"
- },
- "created_at": {
- "type": "string"
- },
- "domains": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "hashtags": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "id": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "links": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "media": {
- "dynamic": "true",
- "properties": {
- "display_url": {
- "type": "string"
- },
- "expanded_url": {
- "type": "string"
- },
- "id": {
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "media_url": {
- "type": "string"
- },
- "media_url_https": {
- "type": "string"
- },
- "sizes": {
- "dynamic": "true",
- "properties": {
- "large": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "integer"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "integer"
- }
- }
- },
- "medium": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "integer"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "integer"
- }
- }
- },
- "small": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "integer"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "integer"
- }
- }
- },
- "thumb": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "integer"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "integer"
- }
- }
- }
- }
- },
- "source_status_id": {
- "type": "integer"
- },
- "source_status_id_str": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "mention_ids": {
- "type": "array",
- "items": [
- {
- "type": "integer"
- }
- ]
- },
- "mentions": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "source": {
- "type": "string"
- },
- "text": {
- "type": "string"
- },
- "user": {
- "type": "DatasiftTwitterUser",
- "$ref": "./DatasiftTwitterUser.json"
- }
- }
- },
- "retweeted": {
- "javaType": "org.apache.streams.datasift.twitter.Retweeted",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "created_at": {
- "type": "string"
- },
- "geo": {
- "dynamic": "true",
- "properties": {
- "latitude": {
- "type": "number"
- },
- "longitude": {
- "type": "number"
- }
- }
- },
- "id": {
- "type": "string"
- },
- "place": {
- "dynamic": "true",
- "properties": {
- "attributes": {
- "type": "object",
- "dynamic": "true"
- },
- "country": {
- "type": "string"
- },
- "country_code": {
- "type": "string"
- },
- "full_name": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "place_type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "source": {
- "type": "string"
- },
- "user": {
- "type": "DatasiftTwitterUser",
- "$ref": "./DatasiftTwitterUser.json"
- }
- }
- },
- "source": {
- "type": "string"
- },
- "text": {
- "type": "string"
- },
- "user": {
- "type": "DatasiftTwitterUser",
- "$ref": "./DatasiftTwitterUser.json"
- }
- }
- },
- "youtube": {
- "javaType": "org.apache.streams.datasift.youtube.YouTube",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "javaType": "org.apache.streams.datasift.youtube.Author",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "commentslink": {
- "type": "string"
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "videolink": {
- "type": "string"
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
deleted file mode 100644
index 9a0f027..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
+++ /dev/null
@@ -1,133 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "javaType": "org.apache.streams.datasift.DatasiftConfiguration",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "apiKey": {
- "type": "string"
- },
- "userName": {
- "type": "string"
- },
- "streamHash": {
- "type": "array",
- "minItems": 0,
- "items": {
- "type": "string"
- }
- },
- "streamCsdl": {
- "type": "array",
- "minItems": 0,
- "items": {
- "type": "string"
- }
- },
- "managedSources": {
- "type": "array",
- "minItems": 0,
- "items": {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSource",
- "type": "object",
- "properties": {
- "auth": {
- "type": "array",
- "items": {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceAuth",
- "type": "object",
- "properties": {
- "identity_id": {
- "type": "string"
- },
- "parameters": {
- "type": "object",
- "properties": {
- "value": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "created_at": {
- "type": "number"
- },
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "parameters": {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceParams",
- "type": "object",
- "properties": {
- "comments": {
- "type": "boolean"
- },
- "likes": {
- "type": "boolean"
- }
- }
- },
- "resources": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceResource",
- "type": "object",
- "properties": {
- "parameters": {
- "type": "object",
- "properties": {
- "distance": {
- "type": "number"
- },
- "exact_match": {
- "type": "string"
- },
- "foursq": {
- "type": "string"
- },
- "lat": {
- "type": "number"
- },
- "lng": {
- "type": "number"
- },
- "type": {
- "type": "string",
- "enum" : ["user", "tag", "area", "location", "popular"]
- },
- "usermeta": {
- "type": "string"
- },
- "value": {
- "type": "number"
- }
- }
- },
- "resource_id": {
- "type": "string"
- }
- }
- }
- ]
- },
- "source_type": {
- "type": "string",
- "enum" : ["facebook_page", "googleplus", "instagram", "yammer"]
- },
- "status": {
- "type": "string"
- }
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftPushConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftPushConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftPushConfiguration.json
deleted file mode 100644
index bb65ef0..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftPushConfiguration.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "javaType" : "org.apache.streams.datasift.DatasiftPushConfiguration",
- "extends": {"$ref":"DatasiftConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "streamHash": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftStreamConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftStreamConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftStreamConfiguration.json
deleted file mode 100644
index 91a9974..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftStreamConfiguration.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "javaType" : "org.apache.streams.datasift.DatasiftStreamConfiguration",
- "extends": {"$ref":"DatasiftConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "streamHash": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftTwitterUser.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftTwitterUser.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftTwitterUser.json
deleted file mode 100644
index 3be49ff..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftTwitterUser.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "javaType": "org.apache.streams.datasift.twitter.DatasiftTwitterUser",
- "properties": {
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "description": {
- "type": "string"
- },
- "favourites_count": {
- "type": "integer"
- },
- "followers_count": {
- "type": "integer"
- },
- "friends_count": {
- "type": "integer"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "id": {
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "listed_count": {
- "type": "integer"
- },
- "location": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "profile_image_url": {
- "type": "string"
- },
- "profile_image_url_https": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "statuses_count": {
- "type": "integer"
- },
- "time_zone": {
- "type": "string"
- },
- "url": {
- "type": "string"
- },
- "utc_offset": {
- "type": "integer"
- },
- "verified": {
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/199881d5/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json
new file mode 100644
index 0000000..d336d73
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json
@@ -0,0 +1,458 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "javaType": "org.apache.streams.datasift.Datasift",
+ "properties": {
+ "blog": {
+ "javaType": "org.apache.streams.datasift.blog.Blog",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "author": {
+ "type": "object",
+ "javaType": "org.apache.streams.datasift.blog.BlogAuthor",
+ "dynamic": "true",
+ "properties": {
+ "link": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ }
+ }
+ },
+ "content": {
+ "type": "string"
+ },
+ "contenttype": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string",
+ "format": "date-time"
+ },
+ "domain": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "link": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ },
+ "demographic": {
+ "dynamic": "true",
+ "properties": {
+ "gender": {
+ "type": "string"
+ }
+ }
+ },
+ "facebook": {
+ "type": "object",
+ "$ref": "facebook/DatasiftFacebook.json"
+ },
+ "instagram": {
+ "type": "object",
+ "$ref": "instagram/DatasiftInstagram.json"
+ },
+ "interaction": {
+ "type": "object",
+ "$ref": "interaction/DatasiftInteraction.json"
+ },
+ "klout": {
+ "javaType": "org.apache.streams.datasift.interaction.Klout",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "score": {
+ "type": "integer"
+ }
+ }
+ },
+ "language": {
+ "javaType": "org.apache.streams.datasift.language.Language",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "confidence": {
+ "type": "integer"
+ },
+ "tag": {
+ "type": "string"
+ },
+ "tag_extended": {
+ "type": "string"
+ }
+ }
+ },
+ "links": {
+ "javaType": "org.apache.streams.datasift.links.Links",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "code": {
+ "type": "array",
+ "items": [
+ {
+ "type": "integer"
+ }
+ ]
+ },
+ "created_at": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "hops": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "meta": {
+ "dynamic": "true",
+ "javaType": "org.apache.streams.datasift.links.Meta",
+ "type": "object",
+ "properties": {
+ "charset": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "content_type": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "description": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "keywords": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "lang": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "opengraph": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "org.apache.streams.datasift.interaction.meta.OpenGraph",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "description": {
+ "type": "string"
+ },
+ "image": {
+ "type": "string"
+ },
+ "site_name": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "twitter": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "org.apache.streams.datasift.interaction.meta.Twitter",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "card": {
+ "type": "string"
+ },
+ "creator": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "image": {
+ "type": "string"
+ },
+ "player": {
+ "type": "string"
+ },
+ "player_height": {
+ "type": "string"
+ },
+ "player_width": {
+ "type": "string"
+ },
+ "site": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ "normalized_url": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "retweet_count": {
+ "type": "array",
+ "items": [
+ {
+ "type": "long"
+ }
+ ]
+ },
+ "title": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "url": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ }
+ }
+ },
+ "salience": {
+ "javaType": "org.apache.streams.datasift.salience.Salience",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "content": {
+ "javaType": "org.apache.streams.datasift.salience.content.Content",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "entities": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "org.apache.streams.datasift.salience.content.Entities",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "about": {
+ "type": "integer"
+ },
+ "confident": {
+ "type": "integer"
+ },
+ "evidence": {
+ "type": "integer"
+ },
+ "label": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "sentiment": {
+ "type": "integer"
+ },
+ "themes": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "sentiment": {
+ "type": "integer"
+ }
+ }
+ },
+ "title": {
+ "javaType": "org.apache.streams.datasift.salience.title.Title",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "entities": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "org.apache.streams.datasift.salience.title.Entities",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "about": {
+ "type": "integer"
+ },
+ "confident": {
+ "type": "integer"
+ },
+ "evidence": {
+ "type": "integer"
+ },
+ "label": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "sentiment": {
+ "type": "integer"
+ },
+ "themes": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "sentiment": {
+ "type": "integer"
+ }
+ }
+ }
+ }
+ },
+ "trends": {
+ "javaType": "org.apache.streams.datasift.trends.Trends",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "content": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "source": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "type": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ }
+ }
+ },
+ "twitter": {
+ "type": "object",
+ "$ref": "twitter/DatasiftTwitter.json"
+ },
+ "youtube": {
+ "javaType": "org.apache.streams.datasift.youtube.YouTube",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "author": {
+ "javaType": "org.apache.streams.datasift.youtube.Author",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "link": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ }
+ }
+ },
+ "commentslink": {
+ "type": "string"
+ },
+ "content": {
+ "type": "string"
+ },
+ "contenttype": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ },
+ "videolink": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file