You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sm...@apache.org on 2017/02/08 04:32:50 UTC
[4/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java
new file mode 100644
index 0000000..091552f
--- /dev/null
+++ b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.sysomos.test;
+
+import com.fasterxml.aalto.stax.InputFactoryImpl;
+import com.fasterxml.aalto.stax.OutputFactoryImpl;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule;
+import com.fasterxml.jackson.dataformat.xml.XmlFactory;
+import com.fasterxml.jackson.dataformat.xml.XmlMapper;
+import com.sysomos.xml.BeatApi;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Tests ability to convert String xml form to {@link com.sysomos.xml.BeatApi} form
+ */
+public class SysomosXmlSerDeIT {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SysomosXmlSerDeIT.class);
+
+ private XmlMapper xmlMapper;
+
+ /**
+ * before.
+ */
+ @Before
+ public void before() {
+
+ XmlFactory xmlFactory = new XmlFactory(new InputFactoryImpl(),
+ new OutputFactoryImpl());
+
+ JacksonXmlModule module = new JacksonXmlModule();
+
+ module.setDefaultUseWrapper(false);
+
+ xmlMapper = new XmlMapper(xmlFactory, module);
+
+ xmlMapper
+ .configure(
+ DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY,
+ Boolean.TRUE);
+ xmlMapper
+ .configure(
+ DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,
+ Boolean.TRUE);
+ xmlMapper
+ .configure(
+ DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY,
+ Boolean.TRUE);
+ xmlMapper.configure(
+ DeserializationFeature.READ_ENUMS_USING_TO_STRING,
+ Boolean.TRUE);
+
+ }
+
+ @Test
+ public void test() {
+
+ InputStream is = SysomosXmlSerDeIT.class.getResourceAsStream("/sysomos_xmls.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ LOGGER.debug(line);
+
+ BeatApi ser = xmlMapper.readValue(line, BeatApi.class);
+
+ String des = xmlMapper.writeValueAsString(ser);
+ LOGGER.debug(des);
+ }
+ } catch ( Exception ex ) {
+ ex.printStackTrace();
+ Assert.fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json
deleted file mode 100644
index 03b801d..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.Delete",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "delete": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.DeleteDetails",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "status": {
- "type": "object",
- "properties": {
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "user_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "user_id_str": {
- "type": "string"
- }
- }
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json
deleted file mode 100644
index 320db12..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.Follow",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "follower": {
- "$ref": "User.json"
- },
- "followee": {
- "$ref": "User.json"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
deleted file mode 100644
index ae7a021..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.FriendList",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "friends": {
- "type": "array",
- "items": {
- "type": "integer"
- }
- },
- "friends_str": {
- "type": "array",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json
deleted file mode 100644
index a449525..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.Retweet",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {"$ref":"tweet.json"},
- "properties": {
- "retweeted_status": {
- "type": "object",
- "required" : false,
- "description" : "Describes the tweet being retweeted.",
- "$ref" : "tweet.json"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json
deleted file mode 100644
index 69048d1..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json
+++ /dev/null
@@ -1,87 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "The protocol",
- "default": "https"
- },
- "host": {
- "type": "string",
- "description": "The host",
- "default": "api.twitter.com"
- },
- "port": {
- "type": "integer",
- "description": "The port",
- "default": 443
- },
- "version": {
- "type": "string",
- "description": "The version",
- "default": "1.1"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint"
- },
- "jsonStoreEnabled": {
- "default" : true,
- "type": "string"
- },
- "oauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.twitter.TwitterOAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "appName": {
- "type": "string"
- },
- "consumerKey": {
- "type": "string"
- },
- "consumerSecret": {
- "type": "string"
- },
- "accessToken": {
- "type": "string"
- },
- "accessTokenSecret": {
- "type": "string"
- }
- }
- },
- "basicauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.twitter.TwitterBasicAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "username": {
- "type": "string"
- },
- "password": {
- "type": "string"
- }
- }
- },
- "retrySleepMs": {
- "type": "integer",
- "description": "ms to sleep when hitting a rate limit",
- "default": 100000
- },
- "retryMax": {
- "type": "integer",
- "description": "ms to sleep when hitting a rate limit",
- "default": 10
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json
deleted file mode 100644
index 89fc7af..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterFollowingConfiguration",
- "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "ids_only": {
- "type": "boolean",
- "description": "Whether to collect ids only, or full profiles",
- "default": "true"
- },
- "max_items": {
- "type": "integer",
- "description": "Max items per user to collect",
- "default": 50000
- },
- "max_pages": {
- "type": "integer",
- "description": "Max pages per user to request",
- "default": 10
- },
- "page_size": {
- "type": "integer",
- "description": "Max items per page to request",
- "default": 5000
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json
deleted file mode 100644
index 6fa2a73..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterStreamConfiguration",
- "extends": {"$ref":"TwitterConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "includeEntities": {
- "type": "string"
- },
- "truncated": {
- "type": "boolean"
- },
- "filter-level": {
- "type": "string",
- "description": "Setting this parameter to one of none, low, or medium will set the minimum value of the filter_level Tweet attribute required to be included in the stream"
- },
- "with": {
- "type": "string",
- "description": "Typically following or user"
- },
- "replies": {
- "type": "string",
- "description": "Set to all, to see all @replies"
- },
- "follow": {
- "type": "array",
- "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
- "items": {
- "type": "integer"
- }
- },
- "track": {
- "type": "array",
- "description": "A list of phrases which will be used to determine what Tweets will be delivered on the stream",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json
deleted file mode 100644
index 37ed60e..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterTimelineProviderConfiguration",
- "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "max_items": {
- "type": "integer",
- "description": "Max items per user to collect",
- "default": 3200
- },
- "max_pages": {
- "type": "integer",
- "description": "Max items per page to request",
- "default": 16
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json
deleted file mode 100644
index 405c87a..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterUserInformationConfiguration",
- "extends": {"$ref":"TwitterConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "info": {
- "type": "array",
- "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
- "items": {
- "type": "string"
- }
- },
- "page_size": {
- "type": "integer",
- "description": "Max items per page to request",
- "default": 200
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json
deleted file mode 100644
index c6cb798..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json
+++ /dev/null
@@ -1,129 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.User",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "location": {
- "type": "string"
- },
- "default_profile": {
- "type": "boolean"
- },
- "statuses_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "profile_background_tile": {
- "type": "boolean"
- },
- "lang": {
- "type": "string"
- },
- "profile_link_color": {
- "type": "string"
- },
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "protected": {
- "type": "boolean"
- },
- "favourites_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "profile_text_color": {
- "type": "string"
- },
- "verified": {
- "type": "boolean"
- },
- "description": {
- "type": "string"
- },
- "contributors_enabled": {
- "type": "boolean"
- },
- "name": {
- "type": "string"
- },
- "profile_sidebar_border_color": {
- "type": "string"
- },
- "profile_background_color": {
- "type": "string"
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "default_profile_image": {
- "type": "boolean"
- },
- "followers_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "profile_image_url_https": {
- "type": "string"
- },
- "profile_background_image_url": {
- "type": "string"
- },
- "profile_background_image_url_https": {
- "type": "string"
- },
- "follow_request_sent": {
- "type": "boolean"
- },
- "url": {
- "type": "string"
- },
- "utc_offset": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "time_zone": {
- "type": "string"
- },
- "profile_use_background_image": {
- "type": "boolean"
- },
- "friends_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "profile_sidebar_fill_color": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "id_str": {
- "type": "string"
- },
- "profile_image_url": {
- "type": "string"
- },
- "is_translator": {
- "type": "boolean"
- },
- "listed_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "status": {
- "$ref": "tweet.json"
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
deleted file mode 100644
index 23f8365..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
+++ /dev/null
@@ -1,52 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.UserstreamEvent",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "created_at": {
- "type": "string",
- "format" : "date-time"
- },
- "event_type": {
- "type": "string",
- "enum" : [
- "access_revoked",
- "block",
- "unblock",
- "favorite",
- "unfavorite",
- "follow",
- "unfollow",
- "list_created",
- "list_destroyed",
- "list_updated",
- "list_member_added",
- "list_member_removed",
- "list_user_subscribed",
- "list_user_unsubscribed",
- "user_update"
- ]
- },
- "source": {
- "type": "string",
- "items": {
- "type": "integer"
- }
- },
- "target": {
- "type": "string",
- "items": {
- "type": "integer"
- }
- },
- "target_object": {
- "type": "object"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
deleted file mode 100644
index e529559..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
+++ /dev/null
@@ -1,219 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.Tweet",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "text": {
- "type": "string"
- },
- "retweeted": {
- "type": "boolean"
- },
- "in_reply_to_screen_name": {
- "type": "string"
- },
- "truncated": {
- "type": "boolean"
- },
- "filter_level": {
- "type": "string"
- },
- "contributors": {
- "ignore_malformed": false,
- "type": "array",
- "items": [{
- "type" : "number"
- }]
- },
- "place": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.Place",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "string"
- },
- "bounding_box": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.BoundingBox",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "type": {
- "type": "string"
- },
- "coordinates": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "type": "number"
- }
- }
- }
- }
- }
- }
- }
- },
- "entities": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.twitter.pojo.Entities",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "user_mentions": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.UserMentions",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "name": {
- "type": "string"
- },
- "indices": {
- "type": "array",
- "items": [{
- "type" : "integer"
- }]
- },
- "screen_name": {
- "type": "string"
- },
- "id_str": {
- "type": "string"
- }
- }
- }
- },
- "hashtags": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.pojo.Hashtag",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "text": {
- "type": "string"
- },
- "indices": {
- "type": "array",
- "items": [{
- "type": "integer"
- }]
- }
- }
- }
- },
- "urls": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.Url",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "expanded_url": {
- "type": "string"
- },
- "indices": {
- "type": "array",
- "items": [
- {
- "type" : "integer"
- }
- ]
- },
- "display_url": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "in_reply_to_status_id_str": {
- "type": "string"
- },
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "in_reply_to_user_id_str": {
- "type": "string"
- },
- "source": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "favorited": {
- "type": "boolean"
- },
- "possibly_sensitive": {
- "type": "boolean"
- },
- "in_reply_to_status_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "created_at": {
- "type" : "string",
- "format" : "date-time"
- },
- "in_reply_to_user_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "favorite_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "retweet_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "user": {
- "$ref": "User.json"
- },
- "is_quote_status": {
- "type": "boolean"
- },
- "quoted_status_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "quoted_status": {
- "type": "object",
- "required" : false,
- "description" : "Describes the tweet being quoted.",
- "$ref" : "tweet.json"
- },
- "retweeted_status_id": {
- "type": "integer"
- },
- "retweeted_status": {
- "type": "object",
- "required" : false,
- "description" : "Describes the tweet being retweeted.",
- "$ref" : "tweet.json"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json
new file mode 100644
index 0000000..69048d1
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json
@@ -0,0 +1,87 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "protocol": {
+ "type": "string",
+ "description": "The protocol",
+ "default": "https"
+ },
+ "host": {
+ "type": "string",
+ "description": "The host",
+ "default": "api.twitter.com"
+ },
+ "port": {
+ "type": "integer",
+ "description": "The port",
+ "default": 443
+ },
+ "version": {
+ "type": "string",
+ "description": "The version",
+ "default": "1.1"
+ },
+ "endpoint": {
+ "type": "string",
+ "description": "The endpoint"
+ },
+ "jsonStoreEnabled": {
+ "default" : true,
+ "type": "string"
+ },
+ "oauth": {
+ "type": "object",
+ "dynamic": "true",
+ "javaType" : "org.apache.streams.twitter.TwitterOAuthConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "appName": {
+ "type": "string"
+ },
+ "consumerKey": {
+ "type": "string"
+ },
+ "consumerSecret": {
+ "type": "string"
+ },
+ "accessToken": {
+ "type": "string"
+ },
+ "accessTokenSecret": {
+ "type": "string"
+ }
+ }
+ },
+ "basicauth": {
+ "type": "object",
+ "dynamic": "true",
+ "javaType" : "org.apache.streams.twitter.TwitterBasicAuthConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "username": {
+ "type": "string"
+ },
+ "password": {
+ "type": "string"
+ }
+ }
+ },
+ "retrySleepMs": {
+ "type": "integer",
+ "description": "ms to sleep when hitting a rate limit",
+ "default": 100000
+ },
+ "retryMax": {
+ "type": "integer",
+ "description": "ms to sleep when hitting a rate limit",
+ "default": 10
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json
new file mode 100644
index 0000000..89fc7af
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json
@@ -0,0 +1,33 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterFollowingConfiguration",
+ "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "ids_only": {
+ "type": "boolean",
+ "description": "Whether to collect ids only, or full profiles",
+ "default": "true"
+ },
+ "max_items": {
+ "type": "integer",
+ "description": "Max items per user to collect",
+ "default": 50000
+ },
+ "max_pages": {
+ "type": "integer",
+ "description": "Max pages per user to request",
+ "default": 10
+ },
+ "page_size": {
+ "type": "integer",
+ "description": "Max items per page to request",
+ "default": 5000
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json
new file mode 100644
index 0000000..6fa2a73
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json
@@ -0,0 +1,45 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterStreamConfiguration",
+ "extends": {"$ref":"TwitterConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "includeEntities": {
+ "type": "string"
+ },
+ "truncated": {
+ "type": "boolean"
+ },
+ "filter-level": {
+ "type": "string",
+ "description": "Setting this parameter to one of none, low, or medium will set the minimum value of the filter_level Tweet attribute required to be included in the stream"
+ },
+ "with": {
+ "type": "string",
+ "description": "Typically following or user"
+ },
+ "replies": {
+ "type": "string",
+ "description": "Set to all, to see all @replies"
+ },
+ "follow": {
+ "type": "array",
+ "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "track": {
+ "type": "array",
+ "description": "A list of phrases which will be used to determine what Tweets will be delivered on the stream",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json
new file mode 100644
index 0000000..37ed60e
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json
@@ -0,0 +1,23 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterTimelineProviderConfiguration",
+ "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "max_items": {
+ "type": "integer",
+ "description": "Max items per user to collect",
+ "default": 3200
+ },
+ "max_pages": {
+ "type": "integer",
+ "description": "Max items per page to request",
+ "default": 16
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json
new file mode 100644
index 0000000..405c87a
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json
@@ -0,0 +1,25 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterUserInformationConfiguration",
+ "extends": {"$ref":"TwitterConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "info": {
+ "type": "array",
+ "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
+ "items": {
+ "type": "string"
+ }
+ },
+ "page_size": {
+ "type": "integer",
+ "description": "Max items per page to request",
+ "default": 200
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json
new file mode 100644
index 0000000..03b801d
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json
@@ -0,0 +1,37 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.Delete",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "delete": {
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.DeleteDetails",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "status": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "user_id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "user_id_str": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json
new file mode 100644
index 0000000..320db12
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json
@@ -0,0 +1,18 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.Follow",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "follower": {
+ "$ref": "User.json"
+ },
+ "followee": {
+ "$ref": "User.json"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json
new file mode 100644
index 0000000..ae7a021
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json
@@ -0,0 +1,24 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.FriendList",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "friends": {
+ "type": "array",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "friends_str": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json
new file mode 100644
index 0000000..a449525
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json
@@ -0,0 +1,19 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.Retweet",
+ "javaInterfaces": ["java.io.Serializable"],
+ "extends": {"$ref":"tweet.json"},
+ "properties": {
+ "retweeted_status": {
+ "type": "object",
+ "required" : false,
+ "description" : "Describes the tweet being retweeted.",
+ "$ref" : "tweet.json"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json
new file mode 100644
index 0000000..c6cb798
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json
@@ -0,0 +1,129 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.User",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "location": {
+ "type": "string"
+ },
+ "default_profile": {
+ "type": "boolean"
+ },
+ "statuses_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "profile_background_tile": {
+ "type": "boolean"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "profile_link_color": {
+ "type": "string"
+ },
+ "id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "protected": {
+ "type": "boolean"
+ },
+ "favourites_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "profile_text_color": {
+ "type": "string"
+ },
+ "verified": {
+ "type": "boolean"
+ },
+ "description": {
+ "type": "string"
+ },
+ "contributors_enabled": {
+ "type": "boolean"
+ },
+ "name": {
+ "type": "string"
+ },
+ "profile_sidebar_border_color": {
+ "type": "string"
+ },
+ "profile_background_color": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string",
+ "format": "date-time"
+ },
+ "default_profile_image": {
+ "type": "boolean"
+ },
+ "followers_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "geo_enabled": {
+ "type": "boolean"
+ },
+ "profile_image_url_https": {
+ "type": "string"
+ },
+ "profile_background_image_url": {
+ "type": "string"
+ },
+ "profile_background_image_url_https": {
+ "type": "string"
+ },
+ "follow_request_sent": {
+ "type": "boolean"
+ },
+ "url": {
+ "type": "string"
+ },
+ "utc_offset": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "time_zone": {
+ "type": "string"
+ },
+ "profile_use_background_image": {
+ "type": "boolean"
+ },
+ "friends_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "profile_sidebar_fill_color": {
+ "type": "string"
+ },
+ "screen_name": {
+ "type": "string"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "profile_image_url": {
+ "type": "string"
+ },
+ "is_translator": {
+ "type": "boolean"
+ },
+ "listed_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "status": {
+ "$ref": "tweet.json"
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json
new file mode 100644
index 0000000..23f8365
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json
@@ -0,0 +1,52 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.UserstreamEvent",
+ "javaInterfaces": ["java.io.Serializable"],
+ "description": "",
+ "properties": {
+ "created_at": {
+ "type": "string",
+ "format" : "date-time"
+ },
+ "event_type": {
+ "type": "string",
+ "enum" : [
+ "access_revoked",
+ "block",
+ "unblock",
+ "favorite",
+ "unfavorite",
+ "follow",
+ "unfollow",
+ "list_created",
+ "list_destroyed",
+ "list_updated",
+ "list_member_added",
+ "list_member_removed",
+ "list_user_subscribed",
+ "list_user_unsubscribed",
+ "user_update"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "target": {
+ "type": "string",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "target_object": {
+ "type": "object"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json
new file mode 100644
index 0000000..e529559
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json
@@ -0,0 +1,219 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.Tweet",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "text": {
+ "type": "string"
+ },
+ "retweeted": {
+ "type": "boolean"
+ },
+ "in_reply_to_screen_name": {
+ "type": "string"
+ },
+ "truncated": {
+ "type": "boolean"
+ },
+ "filter_level": {
+ "type": "string"
+ },
+ "contributors": {
+ "ignore_malformed": false,
+ "type": "array",
+ "items": [{
+ "type" : "number"
+ }]
+ },
+ "place": {
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.Place",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "id": {
+ "type": "string"
+ },
+ "bounding_box": {
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.BoundingBox",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "type": {
+ "type": "string"
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "entities": {
+ "type": "object",
+ "dynamic": "true",
+ "javaType" : "org.apache.streams.twitter.pojo.Entities",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "user_mentions": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.UserMentions",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "indices": {
+ "type": "array",
+ "items": [{
+ "type" : "integer"
+ }]
+ },
+ "screen_name": {
+ "type": "string"
+ },
+ "id_str": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "hashtags": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "javaType": "org.apache.streams.twitter.pojo.Hashtag",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "text": {
+ "type": "string"
+ },
+ "indices": {
+ "type": "array",
+ "items": [{
+ "type": "integer"
+ }]
+ }
+ }
+ }
+ },
+ "urls": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "javaType": "org.apache.streams.twitter.Url",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "expanded_url": {
+ "type": "string"
+ },
+ "indices": {
+ "type": "array",
+ "items": [
+ {
+ "type" : "integer"
+ }
+ ]
+ },
+ "display_url": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ },
+ "in_reply_to_status_id_str": {
+ "type": "string"
+ },
+ "id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "in_reply_to_user_id_str": {
+ "type": "string"
+ },
+ "source": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "favorited": {
+ "type": "boolean"
+ },
+ "possibly_sensitive": {
+ "type": "boolean"
+ },
+ "in_reply_to_status_id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "created_at": {
+ "type" : "string",
+ "format" : "date-time"
+ },
+ "in_reply_to_user_id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "favorite_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "retweet_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "User.json"
+ },
+ "is_quote_status": {
+ "type": "boolean"
+ },
+ "quoted_status_id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "quoted_status": {
+ "type": "object",
+ "required" : false,
+ "description" : "Describes the tweet being quoted.",
+ "$ref" : "tweet.json"
+ },
+ "retweeted_status_id": {
+ "type": "integer"
+ },
+ "retweeted_status": {
+ "type": "object",
+ "required" : false,
+ "description" : "Describes the tweet being retweeted.",
+ "$ref" : "tweet.json"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java
deleted file mode 100644
index d8967d1..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.youtube.model.Channel;
-import com.google.api.services.youtube.model.Video;
-import com.youtube.serializer.YoutubeActivityUtil;
-import com.youtube.serializer.YoutubeChannelDeserializer;
-import com.youtube.serializer.YoutubeEventClassifier;
-import com.youtube.serializer.YoutubeVideoDeserializer;
-import org.apache.commons.lang.NotImplementedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-
-public class YoutubeTypeConverter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "YoutubeTypeConverter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeTypeConverter.class);
-
- private StreamsJacksonMapper mapper;
- private Queue<Video> inQueue;
- private Queue<StreamsDatum> outQueue;
- private int count = 0;
-
- public YoutubeTypeConverter() {}
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum streamsDatum) {
- StreamsDatum result = null;
-
- try {
- Object item = streamsDatum.getDocument();
-
- LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
- Activity activity;
-
- if (item instanceof String) {
- item = deserializeItem(item);
- }
-
- if (item instanceof Video) {
- activity = new Activity();
- YoutubeActivityUtil.updateActivity((Video)item, activity, streamsDatum.getId());
- } else if (item instanceof Channel) {
- activity = new Activity();
- YoutubeActivityUtil.updateActivity((Channel)item, activity, null);
- } else {
- throw new NotImplementedException("Type conversion not implement for type : " + item.getClass().getName());
- }
-
- if (activity != null) {
- result = new StreamsDatum(activity);
- count++;
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while converting Video to Activity: {}", ex);
- }
-
- if ( result != null ) {
- List<StreamsDatum> streamsDatumList = new ArrayList<>();
- streamsDatumList.add(result);
- return streamsDatumList;
- } else {
- return new ArrayList<>();
- }
- }
-
- private Object deserializeItem(Object item) {
- try {
- Class klass = YoutubeEventClassifier.detectClass((String) item);
- if (klass.equals(Video.class)) {
- item = mapper.readValue((String) item, Video.class);
- } else if (klass.equals(Channel.class)) {
- item = mapper.readValue((String) item, Channel.class);
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserializeItem: {}", ex);
- }
-
- return item;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
-
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
- mapper.registerModule(simpleModule);
- simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Channel.class, new YoutubeChannelDeserializer());
- mapper.registerModule(simpleModule);
- }
-
- @Override
- public void cleanUp() {}
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java
deleted file mode 100644
index fd238db..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.Channel;
-import com.google.gson.Gson;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Collects YoutubeChannelData on behalf of YoutubeChannelProvider.
- */
-public class YoutubeChannelDataCollector extends YoutubeDataCollector {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeChannelDataCollector.class);
- private static final String CONTENT = "snippet,contentDetails,statistics,topicDetails";
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- private static final int MAX_ATTEMPTS = 5;
-
- private YouTube youTube;
- private BlockingQueue<StreamsDatum> queue;
- private BackOffStrategy strategy;
- private UserInfo userInfo;
- private YoutubeConfiguration youtubeConfig;
-
- /**
- * YoutubeChannelDataCollector constructor.
- * @param youTube YouTube
- * @param queue BlockingQueue of StreamsDatum
- * @param strategy BackOffStrategy
- * @param userInfo UserInfo
- * @param youtubeConfig YoutubeConfiguration
- */
- public YoutubeChannelDataCollector(
- YouTube youTube,
- BlockingQueue<StreamsDatum> queue,
- BackOffStrategy strategy,
- UserInfo userInfo,
- YoutubeConfiguration youtubeConfig) {
- this.youTube = youTube;
- this.queue = queue;
- this.strategy = strategy;
- this.userInfo = userInfo;
- this.youtubeConfig = youtubeConfig;
- }
-
- @Override
- public void run() {
- Gson gson = new Gson();
- try {
- int attempt = 0;
- YouTube.Channels.List channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setKey(this.youtubeConfig.getApiKey());
- boolean tryAgain = false;
- do {
- try {
- List<Channel> channels = channelLists.execute().getItems();
- for (Channel channel : channels) {
- String json = gson.toJson(channel);
- this.queue.put(new StreamsDatum(json, channel.getId()));
- }
- if (StringUtils.isEmpty(channelLists.getPageToken())) {
- channelLists = null;
- } else {
- channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setOauthToken(this.youtubeConfig.getApiKey())
- .setPageToken(channelLists.getPageToken());
- }
- } catch (GoogleJsonResponseException gjre) {
- LOGGER.warn("GoogleJsonResposneException caught : {}", gjre);
- tryAgain = backoffAndIdentifyIfRetry(gjre, this.strategy);
- ++attempt;
- } catch (Throwable throwable) {
- LOGGER.warn("Unable to get channel info for id : {}", this.userInfo.getUserId());
- LOGGER.warn("Excpection thrown while trying to get channel info : {}", throwable);
- }
- }
- while ((tryAgain && attempt < MAX_ATTEMPTS) || channelLists != null);
-
- } catch (Throwable throwable) {
- LOGGER.warn(throwable.getMessage());
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java
deleted file mode 100644
index d9b1e14..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.services.youtube.YouTube;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.Iterator;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Retrieve recent activity from a list of channels.
- */
-public class YoutubeChannelProvider extends YoutubeProvider {
-
- public YoutubeChannelProvider() {
- super();
- }
-
- public YoutubeChannelProvider(YoutubeConfiguration config) {
- super(config);
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
- return new YoutubeChannelDataCollector(youtube, queue, strategy, userInfo, this.config);
- }
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * youtube.oauth.pathToP12KeyFile
- * youtube.oauth.serviceAccountEmailAddress
- * youtube.apiKey
- * youtube.youtubeUsers
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- YoutubeConfiguration config = new ComponentConfigurator<>(YoutubeConfiguration.class).detectConfiguration(typesafe, "youtube");
- YoutubeChannelProvider provider = new YoutubeChannelProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
- while (iterator.hasNext()) {
- StreamsDatum datum = iterator.next();
- String json;
- try {
- if ( datum.getDocument() instanceof String ) {
- json = (String) datum.getDocument();
- } else {
- json = mapper.writeValueAsString(datum.getDocument());
- }
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java
deleted file mode 100644
index 3eede18..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.util.api.requests.backoff.BackOffException;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base Collector for Youtube Data.
- */
-public abstract class YoutubeDataCollector implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeDataCollector.class);
-
- /**
- * Looks at the status code of the expception. If the code indicates that the request should be retried,
- * it executes the back off strategy and returns true.
- * @param gjre
- * @param backOff
- * @return returns true if the error code of the exception indicates the request should be retried.
- */
- public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException {
- boolean tryAgain = false;
-
- switch (gjre.getStatusCode()) {
- case 400 :
- LOGGER.warn("Bad Request : {}", gjre);
- break;
- case 401 :
- LOGGER.warn("Invalid Credentials : {}", gjre);
- break;
- case 403 :
- LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
- backOff.backOff();
- tryAgain = true;
- break;
- case 503 :
- LOGGER.warn("Google Backend Service Error : {}", gjre);
- break;
- default:
- LOGGER.warn("Google Service returned error : {}", gjre);
- tryAgain = true;
- backOff.backOff();
- break;
- }
-
- return tryAgain;
- }
-}
\ No newline at end of file