You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/10/19 01:06:33 UTC
[1/6] incubator-streams git commit: Resolves STREAMS-416
Repository: incubator-streams
Updated Branches:
refs/heads/master bb2679916 -> 5fd26a677
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/jsonschema/com/gnip/Gnip.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/jsonschema/com/gnip/Gnip.json b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/jsonschema/com/gnip/Gnip.json
deleted file mode 100644
index ec3a468..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/jsonschema/com/gnip/Gnip.json
+++ /dev/null
@@ -1,815 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "com.gnip.pojo.Gnip",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "profileLocations": {
- "properties": {
- "geo": {
- "properties": {
- "type": {
- "type": "string"
- },
- "coordinates": {
- "type": "double"
- }
- }
- },
- "address": {
- "properties": {
- "region": {
- "type": "string"
- },
- "countryCode": {
- "type": "string"
- },
- "locality": {
- "type": "string"
- },
- "country": {
- "type": "string"
- }
- }
- },
- "displayName": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- }
- }
- },
- "klout_score": {
- "type": "long"
- },
- "urls": {
- "properties": {
- "expanded_url": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "language": {
- "properties": {
- "value": {
- "type": "string"
- }
- }
- },
- "body": {
- "type": "string"
- },
- "inReplyTo": {
- "properties": {
- "content": {
- "type": "string"
- },
- "author": {
- "properties": {
- "link": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- }
- }
- },
- "link": {
- "type": "string"
- }
- }
- },
- "tumblrRebloggedRoot": {
- "properties": {
- "author": {
- "properties": {
- "link": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- }
- }
- },
- "link": {
- "type": "string"
- }
- }
- },
- "favoritesCount": {
- "type": "long"
- },
- "geo": {
- "properties": {
- "type": {
- "type": "string"
- },
- "coordinates": {
- "type": "double"
- }
- }
- },
- "location": {
- "javaType": "com.gnip.config.Location",
- "properties": {
- "geo": {
- "properties": {
- "type": {
- "type": "string"
- },
- "coordinates": {
- "type": "double"
- }
- }
- },
- "name": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "twitter_country_code": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- },
- "country_code": {
- "type": "string"
- }
- }
- },
- "link": {
- "type": "string"
- },
- "twitter_lang": {
- "type": "string"
- },
- "retweetCount": {
- "type": "long"
- },
- "postedTime": {
- "type": "string"
- },
- "provider": {
- "javaType": "com.gnip.config.Provider",
- "properties": {
- "link": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- }
- }
- },
- "actor": {
- "javaType": "com.gnip.config.Actor",
- "properties": {
- "twitterTimeZone": {
- "type": "string"
- },
- "summary": {
- "type": "string"
- },
- "friendsCount": {
- "type": "long"
- },
- "favoritesCount": {
- "type": "long"
- },
- "location": {
- "properties": {
- "displayName": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- }
- }
- },
- "link": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "postedTime": {
- "type": "string"
- },
- "links": {
- "properties": {
- "rel": {
- "type": "string"
- },
- "href": {
- "type": "string"
- }
- }
- },
- "listedCount": {
- "type": "long"
- },
- "id": {
- "type": "string"
- },
- "languages": {
- "type": "string"
- },
- "verified": {
- "type": "boolean"
- },
- "utcOffset": {
- "type": "string"
- },
- "followersCount": {
- "type": "long"
- },
- "statusesCount": {
- "type": "long"
- },
- "preferredUsername": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- }
- },
- "object": {
- "javaType": "com.gnip.config.Object",
- "properties": {
- "summary": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "postedTime": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- }
- }
- },
- "actor": {
- "properties": {
- "twitterTimeZone": {
- "type": "string"
- },
- "summary": {
- "type": "string"
- },
- "friendsCount": {
- "type": "long"
- },
- "favoritesCount": {
- "type": "long"
- },
- "location": {
- "properties": {
- "displayName": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- }
- }
- },
- "link": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "postedTime": {
- "type": "string"
- },
- "links": {
- "properties": {
- "rel": {
- "type": "string"
- },
- "href": {
- "type": "string"
- }
- }
- },
- "listedCount": {
- "type": "long"
- },
- "i": {
- "type": "string"
- },
- "languages": {
- "type": "string"
- },
- "verified": {
- "type": "boolean"
- },
- "utcOffset": {
- "type": "string"
- },
- "followersCount": {
- "type": "long"
- },
- "statusesCount": {
- "type": "long"
- },
- "preferredUsername": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- }
- }
- },
- "tumblrFormat": {
- "type": "string"
- },
- "embedCode": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "quoteSource": {
- "type": "string"
- },
- "verb": {
- "type": "string"
- },
- "tumblrReblogKey": {
- "type": "string"
- },
- "tumblrType": {
- "type": "string"
- },
- "generator": {
- "properties": {
- "link": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- }
- }
- },
- "year": {
- "type": "long"
- },
- "artist": {
- "type": "string"
- },
- "albumArt": {
- "type": "string"
- },
- "tags": {
- "properties": {
- "displayName": {
- "type": "string"
- }
- }
- },
- "inReplyTo": {
- "properties": {
- "link": {
- "type": "string"
- }
- }
- },
- "geo": {
- "properties": {
- "type": {
- "type": "string"
- },
- "coordinates": {
- "type": "double"
- }
- }
- },
- "favoritesCount": {
- "type": "long"
- },
- "twitter_lang": {
- "type": "string"
- },
- "image": {
- "properties": {
- "height": {
- "type": "long"
- },
- "width": {
- "type": "long"
- },
- "link": {
- "type": "string"
- }
- }
- },
- "provider": {
- "properties": {
- "link": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- }
- }
- },
- "trackNumber": {
- "type": "string"
- },
- "twitter_entities": {
- "properties": {
- "urls": {
- "properties": {
- "expanded_url": {
- "type": "string"
- },
- "indices": {
- "type": "long"
- },
- "display_url": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "hashtags": {
- "properties": {
- "text": {
- "type": "string"
- },
- "indices": {
- "type": "long"
- }
- }
- },
- "media": {
- "properties": {
- "sizes": {
- "properties": {
- "thumb": {
- "properties": {
- "w": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "h": {
- "type": "long"
- }
- }
- },
- "small": {
- "properties": {
- "w": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "h": {
- "type": "long"
- }
- }
- },
- "medium": {
- "properties": {
- "w": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "h": {
- "type": "long"
- }
- }
- },
- "large": {
- "properties": {
- "w": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "h": {
- "type": "long"
- }
- }
- }
- }
- },
- "id": {
- "type": "long"
- },
- "media_url_https": {
- "type": "string"
- },
- "media_url": {
- "type": "string"
- },
- "expanded_url": {
- "type": "string"
- },
- "source_status_id_str": {
- "type": "string"
- },
- "indices": {
- "type": "long"
- },
- "source_status_id": {
- "type": "long"
- },
- "id_str": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "display_url": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "user_mentions": {
- "properties": {
- "id": {
- "type": "long"
- },
- "name": {
- "type": "string"
- },
- "indices": {
- "type": "long"
- },
- "screen_name": {
- "type": "string"
- },
- "id_str": {
- "type": "string"
- }
- }
- }
- }
- },
- "targetUrl": {
- "type": "string"
- },
- "content": {
- "type": "string"
- },
- "source": {
- "properties": {
- "link": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- }
- }
- },
- "items": {
- "properties": {
- "summary": {
- "type": "string"
- },
- "image": {
- "properties": {
- "height": {
- "type": "long"
- },
- "width": {
- "type": "long"
- },
- "link": {
- "type": "string"
- }
- }
- },
- "objectType": {
- "type": "string"
- },
- "fullImage": {
- "properties": {
- "height": {
- "type": "long"
- },
- "width": {
- "type": "long"
- },
- "link": {
- "type": "string"
- }
- }
- }
- }
- },
- "totalItems": {
- "type": "long"
- },
- "displayName": {
- "type": "string"
- },
- "objectType": {
- "type": "string"
- }
- }
- },
- "twitter_entities": {
- "properties": {
- "urls": {
- "properties": {
- "expanded_url": {
- "type": "string"
- },
- "indices": {
- "type": "long"
- },
- "display_url": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "hashtags": {
- "properties": {
- "text": {
- "type": "string"
- },
- "indices": {
- "type": "long"
- }
- }
- },
- "media": {
- "properties": {
- "sizes": {
- "properties": {
- "thumb": {
- "properties": {
- "w": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "h": {
- "type": "long"
- }
- }
- },
- "small": {
- "properties": {
- "w": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "h": {
- "type": "long"
- }
- }
- },
- "medium": {
- "properties": {
- "w": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "h": {
- "type": "long"
- }
- }
- },
- "large": {
- "properties": {
- "w": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "h": {
- "type": "long"
- }
- }
- }
- }
- },
- "id": {
- "type": "long"
- },
- "media_url_https": {
- "type": "string"
- },
- "media_url": {
- "type": "string"
- },
- "expanded_url": {
- "type": "string"
- },
- "source_status_id_str": {
- "type": "string"
- },
- "indices": {
- "type": "long"
- },
- "source_status_id": {
- "type": "long"
- },
- "id_str": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "display_url": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "user_mentions": {
- "properties": {
- "id": {
- "type": "long"
- },
- "name": {
- "type": "string"
- },
- "indices": {
- "type": "long"
- },
- "screen_name": {
- "type": "string"
- },
- "id_str": {
- "type": "string"
- }
- }
- }
- },
- "tumblrRebloggedFrom": {
- "properties": {
- "author": {
- "properties": {
- "link": {
- "type": "string"
- },
- "displayName": {
- "type": "string"
- }
- }
- },
- "link": {
- "type": "string"
- }
- }
- },
- "twitter_filter_level": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/xmlschema/com/gnip/binding.xjb
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/xmlschema/com/gnip/binding.xjb b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/xmlschema/com/gnip/binding.xjb
deleted file mode 100644
index d94a76b..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/xmlschema/com/gnip/binding.xjb
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<jaxb:bindings xmlns:jaxb="http://java.sun.com/xml/ns/jaxb" xmlns:xs="http://www.w3.org/2001/XMLSchema"
- xmlns:xjc="http://java.sun.com/xml/ns/jaxb/xjc" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://java.sun.com/xml/ns/jaxb http://java.sun.com/xml/ns/jaxb/bindingschema_2_0.xsd"
- version="2.1">
- <jaxb:globalBindings>
- <!-- Use java.util.Calendar instead of javax.xml.datatype.XMLGregorianCalendar for xs:dateTime -->
- <jaxb:javaType name="java.util.Calendar" xmlType="xs:dateTime"
- parseMethod="javax.xml.bind.DatatypeConverter.parseDateTime"
- printMethod="javax.xml.bind.DatatypeConverter.printDateTime" />
-
- <!-- Force all classes implements Serializable -->
- <xjc:serializable uid="1" />
- </jaxb:globalBindings>
-
-</jaxb:bindings>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/xmlschema/com/gnip/entry.xsd
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/xmlschema/com/gnip/entry.xsd b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/xmlschema/com/gnip/entry.xsd
deleted file mode 100644
index da7cf99..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/xmlschema/com/gnip/entry.xsd
+++ /dev/null
@@ -1,398 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<xs:schema version="1.0" xmlns:xs="http://www.w3.org/2001/XMLSchema">
-
- <xs:element name="entry" type="entry"/>
-
- <xs:complexType name="entry">
- <xs:sequence>
- <xs:element name="title" type="xs:string" minOccurs="0"/>
- <xs:element name="id" type="xs:string" minOccurs="0"/>
- <!--<xs:element name="fb:id" type="xs:positiveInteger" minOccurs="0"/>-->
- <xs:element name="link" minOccurs="0">
- <xs:complexType>
- <xs:attribute name="rel" type="xs:string"/>
- <xs:attribute name="type" type="xs:string"/>
- <xs:attribute name="href" type="xs:string"/>
- </xs:complexType>
- </xs:element>
- <xs:element name="published" type="xs:string" minOccurs="0"/>
- <xs:element name="updated" type="xs:string" minOccurs="0"/>
- <xs:element name="author" type="author" minOccurs="0"/>
- <xs:element name="category" minOccurs="0">
- <xs:complexType>
- <xs:attribute name="term" type="xs:string"/>
- <xs:attribute name="label" type="xs:string"/>
- </xs:complexType>
- </xs:element>
- <xs:element name="articles" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="article" type="article" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="article">
- <xs:sequence>
- <xs:element name="sequenceId" type="xs:string" minOccurs="0"/>
- <xs:element name="id" type="xs:string"/>
- <xs:element name="language" type="xs:string"/>
- <xs:element name="title" type="xs:string"/>
- <xs:element name="subTitle" type="xs:string" minOccurs="0"/>
- <xs:element name="content" type="xs:string" minOccurs="0"/>
- <xs:element name="contentWithMarkup" type="xs:string" minOccurs="0"/>
- <xs:element name="tags" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="tag" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="publishedDate" type="xs:string"/>
- <xs:element name="harvestDate" type="xs:string"/>
- <xs:element name="embargoDate" type="xs:string" minOccurs="0"/>
- <xs:element name="licenseEndDate" type="xs:string"/>
- <xs:element name="url" type="xs:string"/>
- <xs:element name="originalUrl" type="xs:string" minOccurs="0"/>
- <xs:element name="commentsUrl" type="xs:string"/>
- <xs:element name="outboundUrls" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="outboundUrl" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="wordCount" type="xs:string" minOccurs="0"/>
- <xs:element name="dataFormat" type="xs:string"/>
- <xs:element name="copyright" type="xs:string"/>
- <xs:element name="loginStatus" type="xs:string"/>
- <xs:element name="duplicateGroupId" type="xs:string"/>
- <xs:element name="contentGroupIds" type="xs:string" minOccurs="0"/>
- <xs:element name="harvest" type="harvest" minOccurs="0"/>
- <xs:element name="media" type="media"/>
- <xs:element name="publishingPlatform" type="articlePublishingPlatform"/>
- <xs:element name="adultLanguage" type="xs:string"/>
- <xs:element name="topics" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="topic" type="topic" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="companies" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="company" type="company" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="locations" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="location" type="articleLocation" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="semantics" type="semantics" minOccurs="0"/>
- <xs:element name="print" type="print" minOccurs="0"/>
- <xs:element name="broadcast" type="broadcast" minOccurs="0"/>
- <xs:element name="author" type="author"/>
- <xs:element name="source" type="source"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="harvest">
- <xs:sequence>
- <xs:element name="headlineVia" type="xs:string"/>
- <xs:element name="contentVia" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="media">
- <xs:sequence>
- <xs:element name="audio" type="audioOrVideo" minOccurs="0" maxOccurs="unbounded"/>
- <xs:element name="image" type="image" minOccurs="0" maxOccurs="unbounded"/>
- <xs:element name="video" type="audioOrVideo" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="audioOrVideo">
- <xs:sequence>
- <xs:element name="url" type="xs:string"/>
- <xs:element name="mimeType" type="xs:string"/>
- <xs:element name="caption" type="xs:string"/>
- <xs:element name="duration" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="image">
- <xs:sequence>
- <xs:element name="url" type="xs:string"/>
- <xs:element name="mimeType" type="xs:string"/>
- <xs:element name="caption" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="articlePublishingPlatform">
- <xs:sequence>
- <xs:element name="itemId" type="xs:string"/>
- <xs:element name="statusId" type="xs:string"/>
- <xs:element name="inReplyToUserId" type="xs:string"/>
- <xs:element name="inReplyToStatusId" type="xs:string"/>
- <xs:element name="totalViews" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="topic">
- <xs:sequence>
- <xs:element name="name" type="xs:string" minOccurs="0"/>
- <xs:element name="group" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="company">
- <xs:sequence>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="symbol" type="xs:string"/>
- <xs:element name="exchange" type="xs:string"/>
- <xs:element name="isin" type="xs:string"/>
- <xs:element name="cusip" type="xs:string" minOccurs="0"/>
- <xs:element name="titleCount" type="xs:int"/>
- <xs:element name="contentCount" type="xs:int"/>
- <xs:element name="primary" type="xs:boolean"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="articleLocation">
- <xs:sequence>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="type" type="xs:string"/>
- <xs:element name="class" type="xs:string"/>
- <xs:element name="mentions" type="xs:string"/>
- <xs:element name="confidence" type="xs:string"/>
- <xs:element name="country" type="country"/>
- <xs:element name="region" type="xs:string"/>
- <xs:element name="subregion" type="xs:string"/>
- <xs:element name="state" type="state"/>
- <xs:element name="latitude" type="xs:string"/>
- <xs:element name="longitude" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="country">
- <xs:sequence>
- <xs:element name="confidence" type="xs:string" minOccurs="0"/>
- <xs:element name="fipsCode" type="xs:string" minOccurs="0"/>
- <xs:element name="isoCode" type="xs:string" minOccurs="0"/>
- <xs:element name="name" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="state">
- <xs:sequence>
- <xs:element name="confidence" type="xs:string" minOccurs="0"/>
- <xs:element name="fipsCode" type="xs:string" minOccurs="0"/>
- <xs:element name="name" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="semantics">
- <xs:sequence>
- <xs:element name="events" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="event" type="semanticsItem" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="entities" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="entity" type="semanticsItem" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="semanticsItem">
- <xs:sequence>
- <xs:element name="properties" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="property" type="property" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="property">
- <xs:sequence>
- <xs:element name="name" type="xs:string" minOccurs="0"/>
- <xs:element name="value" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="print">
- <xs:sequence>
- <xs:element name="supplement" type="xs:string"/>
- <xs:element name="publicationEdition" type="xs:string"/>
- <xs:element name="regionalEdition" type="xs:string"/>
- <xs:element name="section" type="xs:string"/>
- <xs:element name="pageNumber" type="xs:string"/>
- <xs:element name="sizeCm" type="xs:string"/>
- <xs:element name="sizePercentage" type="xs:string"/>
- <xs:element name="originLeft" type="xs:string"/>
- <xs:element name="originTop" type="xs:string"/>
- <xs:element name="width" type="xs:string"/>
- <xs:element name="height" type="xs:string"/>
- <xs:element name="byLine" type="xs:string"/>
- <xs:element name="photo" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="broadcast">
- <xs:sequence>
- <xs:element name="marketName" type="xs:string"/>
- <xs:element name="nationalNetwork" type="xs:string"/>
- <xs:element name="title" type="xs:string"/>
- <xs:element name="lines" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="line" type="line" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="line">
- <xs:sequence>
- <xs:element name="date" type="xs:string" minOccurs="0"/>
- <xs:element name="text" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="author">
- <xs:sequence>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="homeUrl" type="xs:string"/>
- <xs:element name="email" type="xs:string"/>
- <xs:element name="description" type="xs:string"/>
- <xs:element name="dateLastActive" type="xs:string"/>
- <xs:element name="publishingPlatform" type="authorPublishingPlatform"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="authorPublishingPlatform">
- <xs:sequence>
- <xs:element name="userName" type="xs:string"/>
- <xs:element name="userId" type="xs:string"/>
- <xs:element name="statusesCount" type="xs:string"/>
- <xs:element name="totalViews" type="xs:string"/>
- <xs:element name="followingCount" type="xs:string"/>
- <xs:element name="followersCount" type="xs:string"/>
- <xs:element name="kloutScore" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="source">
- <xs:sequence>
- <xs:element name="id" type="xs:string" minOccurs="0"/>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="homeUrl" type="xs:string"/>
- <xs:element name="publisher" type="xs:string"/>
- <xs:element name="primaryLanguage" type="xs:string"/>
- <xs:element name="primaryMediaType" type="xs:string"/>
- <xs:element name="category" type="xs:string"/>
- <xs:element name="editorialRank" type="xs:string"/>
- <xs:element name="publicationId" type="xs:string" minOccurs="0"/>
- <xs:element name="channelCode" type="xs:string" minOccurs="0"/>
- <xs:element name="location" type="sourceLocation"/>
- <xs:element name="feedSource" type="xs:string" minOccurs="0"/>
- <xs:element name="feed" type="feed"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="sourceLocation">
- <xs:sequence>
- <xs:element name="country" type="xs:string"/>
- <xs:element name="countryCode" type="xs:string"/>
- <xs:element name="region" type="xs:string"/>
- <xs:element name="subregion" type="xs:string"/>
- <xs:element name="state" type="xs:string"/>
- <xs:element name="zipArea" type="xs:string"/>
- <xs:element name="zipCode" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="feed">
- <xs:sequence>
- <xs:element name="id" type="xs:string"/>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="url" type="xs:string" minOccurs="0"/>
- <xs:element name="mediaType" type="xs:string"/>
- <xs:element name="publishingPlatform" type="xs:string"/>
- <xs:element name="idFromPublisher" type="xs:string"/>
- <xs:element name="generator" type="xs:string"/>
- <xs:element name="description" type="xs:string"/>
- <xs:element name="tags" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="tag" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="imageUrl" type="xs:string"/>
- <xs:element name="copyright" type="xs:string"/>
- <xs:element name="language" type="xs:string"/>
- <xs:element name="dataFormat" type="xs:string"/>
- <xs:element name="rank" type="rank"/>
- <xs:element name="inWhiteList" type="xs:string"/>
- <xs:element name="autoTopics" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="autoTopic" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="editorialTopics" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="editorialTopic" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="genre" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="rank">
- <xs:sequence>
- <xs:element name="autoRank" type="xs:string"/>
- <xs:element name="autoRankOrder" type="xs:string"/>
- <xs:element name="inboundLinkCount" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-</xs:schema>
-
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/site/markdown/index.md b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/site/markdown/index.md
deleted file mode 100644
index 1131a58..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/site/markdown/index.md
+++ /dev/null
@@ -1,15 +0,0 @@
-gnip-powertrack
-==============
-
-gnip-powertrack contains schema definitions, providers, conversions, and utility classes.
-
-## Data Types
-
-| Schema |
-|--------|
-| [Gnip.json](com/gnip/Gnip.json "Gnip.json")
-| [entry.xsd](com/gnip/entry.xsd "entry.xsd")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/test/java/org/apache/streams/gnip/powertrack/test/PowerTrackDeserializationTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/test/java/org/apache/streams/gnip/powertrack/test/PowerTrackDeserializationTest.java b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/test/java/org/apache/streams/gnip/powertrack/test/PowerTrackDeserializationTest.java
deleted file mode 100644
index 6ff7f6e..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/test/java/org/apache/streams/gnip/powertrack/test/PowerTrackDeserializationTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gnip.powertrack.test;
-
-import org.apache.streams.gnip.powertrack.PowerTrackActivitySerializer;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Created with IntelliJ IDEA.
- * User: rebanks
- * Date: 9/6/13
- * Time: 9:28 AM
- * To change this template use File | Settings | File Templates.
- */
-public class PowerTrackDeserializationTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PowerTrackActivitySerializer.class);
-
- private static final String ptData1= "{\"gnip\":{\"matching_rules\":[{\"tag\":\"toujours_DE_DE_de\"}],\"klout_score\":32,\"klout_profile\":{\"topics\":[],\"klout_user_id\":\"141018977628748348\",\"link\":\"http://klout.com/user/id/141018977628748348\"},\"language\":{\"value\":\"fr\"}},\"body\":\"RT @Albayraakkk: J'attends ton SMS, tu attends le miens. On croit toujours que l'autre va envoyer un message en premier. Bref, on ne se par\\u201a??\",\"favoritesCount\":0,\"link\":\"http://twitter.com/alexandragriett/statuses/375760903104638976\",\"retweetCount\":168,\"twitter_lang\":\"fr\",\"postedTime\":\"2013-09-05T23:22:35.000Z\",\"provider\":{\"link\":\"http://www.twitter.com\",\"displayName\":\"Twitter\",\"objectType\":\"service\"},\"actor\":{\"summary\":null,\"twitterTimeZone\":null,\"friendsCount\":68,\"favoritesCount\":3,\"link\":\"http://www.twitter.com/alexandragriett\",\"postedTime\":\"2013-05-01T17:52:16.000Z\",\"image\":\"https://si0.twimg.com/profile_images/37880000041435
3370/ef4170ca183eed9f7f30665712baba05_normal.jpeg\",\"links\":[{\"rel\":\"me\",\"href\":null}],\"listedCount\":0,\"id\":\"id:twitter.com:1395160326\",\"languages\":[\"fr\"],\"verified\":false,\"utcOffset\":null,\"followersCount\":47,\"preferredUsername\":\"alexandragriett\",\"displayName\":\"MauvaiseIdee\",\"statusesCount\":333,\"objectType\":\"person\"},\"object\":{\"body\":\"J'attends ton SMS, tu attends le miens. On croit toujours que l'autre va envoyer un message en premier. Bref, on ne se parle plus...\",\"favoritesCount\":24,\"link\":\"http://twitter.com/Albayraakkk/statuses/370496182172540928\",\"twitter_lang\":\"fr\",\"postedTime\":\"2013-08-22T10:42:27.000Z\",\"provider\":{\"link\":\"http://www.twitter.com\",\"displayName\":\"Twitter\",\"objectType\":\"service\"},\"actor\":{\"summary\":\"Le meilleur est dans mes favoris ! J'followback sur Instagram : http://instagram.com/Albayraakkk Kik : Cihan69200 #TeamBooba #TeamLacrim\",\"twitterTimeZone\":\"Athens\",\"friendsCount\":24
998,\"favoritesCount\":677,\"location\":{\"displayName\":\"Dans Ta Timeline\",\"objectType\":\"place\"},\"link\":\"http://www.twitter.com/Albayraakkk\",\"postedTime\":\"2012-06-23T20:59:05.000Z\",\"image\":\"https://si0.twimg.com/profile_images/378800000410070574/26edc26ad5ccb223da8b850244b468eb_normal.jpeg\",\"links\":[{\"rel\":\"me\",\"href\":\"http://facebook.com/CihanAlbayraak\"}],\"listedCount\":69,\"id\":\"id:twitter.com:616472380\",\"languages\":[\"fr\"],\"verified\":false,\"utcOffset\":\"10800\",\"followersCount\":76068,\"preferredUsername\":\"Albayraakkk\",\"displayName\":\"LA VIRGULE \\u201a??\",\"statusesCount\":671,\"objectType\":\"person\"},\"object\":{\"summary\":\"J'attends ton SMS, tu attends le miens. On croit toujours que l'autre va envoyer un message en premier. Bref, on ne se parle plus...\",\"id\":\"object:search.twitter.com,2005:370496182172540928\",\"link\":\"http://twitter.com/Albayraakkk/statuses/370496182172540928\",\"postedTime\":\"2013-08-22T10:42:27.000Z
\",\"objectType\":\"note\"},\"twitter_entities\":{\"symbols\":[],\"urls\":[],\"hashtags\":[],\"user_mentions\":[]},\"id\":\"tag:search.twitter.com,2005:370496182172540928\",\"verb\":\"post\",\"generator\":{\"link\":\"http://twitter.com/download/iphone\",\"displayName\":\"Twitter for iPhone\"},\"objectType\":\"activity\"},\"twitter_entities\":{\"symbols\":[],\"urls\":[],\"hashtags\":[],\"user_mentions\":[{\"id\":616472380,\"name\":\"LA VIRGULE \\u201a??\",\"indices\":[3,15],\"screen_name\":\"Albayraakkk\",\"id_str\":\"616472380\"}]},\"twitter_filter_level\":\"medium\",\"content\":\"RT @Albayraakkk: J'attends ton SMS, tu attends le miens. On croit toujours que l'autre va envoyer un message en premier. Bref, on ne se par\\u201a??\",\"id\":\"tag:search.twitter.com,2005:375760903104638976\",\"verb\":\"share\",\"generator\":{\"link\":\"http://twitter.com/download/iphone\",\"displayName\":\"Twitter for iPhone\"},\"published\":\"2013-09-05T23:22:35.000Z\",\"objectType\":\"activity\"}";
-
- @Test
- public void deserializationTest() {
-
- PowerTrackActivitySerializer serializer = new PowerTrackActivitySerializer();
- try {
- Object activity = serializer.deserialize(ptData1);
- } catch( Exception e ) {
- LOGGER.error(e.getMessage(), e );
- e.printStackTrace();
- Assert.fail();
- }
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/pom.xml b/streams-contrib/streams-provider-gnip/pom.xml
deleted file mode 100644
index 79eba48..0000000
--- a/streams-contrib/streams-provider-gnip/pom.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-provider-gnip</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Gnip Providers</description>
-
- <packaging>pom</packaging>
-
- <properties>
-
- </properties>
-
- <modules>
- <module>gnip-powertrack</module>
- <module>gnip-edc-facebook</module>
- <module>gnip-edc-flickr</module>
- <module>gnip-edc-googleplus</module>
- <module>gnip-edc-instagram</module>
- <module>gnip-edc-reddit</module>
- <module>gnip-edc-youtube</module>
- </modules>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
-</project>
[2/6] incubator-streams git commit: Resolves STREAMS-416
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/src/main/java/com/gplus/api/GPlusActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/src/main/java/com/gplus/api/GPlusActivitySerializer.java b/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/src/main/java/com/gplus/api/GPlusActivitySerializer.java
deleted file mode 100644
index 8b04d26..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/src/main/java/com/gplus/api/GPlusActivitySerializer.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.gplus.api;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.gnip.powertrack.GnipActivityFixer;
-import org.apache.streams.pojo.json.Activity;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Created with IntelliJ IDEA.
- * User: mdelaet
- * Date: 8/30/13
- * Time: 10:48 AM
- * To change this template use File | Settings | File Templates.
- */
-public class GPlusActivitySerializer implements ActivitySerializer<String> {
- private final static Logger LOGGER = LoggerFactory.getLogger(GPlusActivitySerializer.class);
-
- @Override
- public String serializationFormat() {
- return "application/gplus+xml";
- }
-
- @Override
- public String serialize(Activity deserialized) {
- ObjectMapper jsonMapper = new ObjectMapper();
- String jsonString = new String();
- try{
- jsonString = jsonMapper.writeValueAsString(deserialized);
- }catch(Exception e){
- LOGGER.error("Exception serializing Activity Object: " + e);
- }
- return jsonString; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public Activity deserialize(String serialized) {
- ObjectMapper jsonMapper = new ObjectMapper();
- JSONObject jsonObject = new JSONObject();
- JSONObject fixedObject = new JSONObject();
-
- try{
- jsonObject = new JSONObject(serialized);
- fixedObject = GnipActivityFixer.fix(jsonObject);
- }catch(Exception e){
- LOGGER.error("Exception deserializing string: " + e);
- }
-
- Activity activity = new Activity();
- try {
- activity = jsonMapper.readValue(fixedObject.toString(), Activity.class);
- Map<String, Object> extension = ExtensionUtil.getInstance().ensureExtensions(activity);
- extension.put("likes", fixedObject.getJSONObject("object").getJSONObject("plusoners").get("totalItems"));
- extension.put("rebroadcast", fixedObject.getJSONObject("object").getJSONObject("resharers").get("totalItems"));
- } catch( Exception e ) {
- LOGGER.error(jsonObject.toString());
- LOGGER.error(fixedObject.toString());
- e.printStackTrace();
- }
- return activity;
- }
-
- @Override
- public List<Activity> deserializeAll(List<String> serializedList) {
- //TODO Support
- throw new NotImplementedException("Not currently supported by this deserializer");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/src/test/java/com/gplus/api/GPlusEDCAsActivityTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/src/test/java/com/gplus/api/GPlusEDCAsActivityTest.java b/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/src/test/java/com/gplus/api/GPlusEDCAsActivityTest.java
deleted file mode 100644
index 10e58e6..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/src/test/java/com/gplus/api/GPlusEDCAsActivityTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.gplus.api;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests conversion of YoutubeEDC inputs to Activity
- */
-@Ignore("ignore until test resources are available.")
-public class GPlusEDCAsActivityTest {
- private final static Logger LOGGER = LoggerFactory.getLogger(GPlusEDCAsActivityTest.class);
-
- private ObjectMapper jsonMapper;
- XmlMapper xmlMapper;
- private GPlusActivitySerializer gplusSerializer;
-
- public GPlusEDCAsActivityTest() {
- gplusSerializer = new GPlusActivitySerializer();
- jsonMapper = new ObjectMapper();
- xmlMapper = new XmlMapper();
- }
-
- @Test
- public void Tests() throws Exception
- {
- InputStream is = GPlusEDCAsActivityTest.class.getResourceAsStream("/GPlusEDCFixed.json");
- if(is == null) LOGGER.debug("null");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- jsonMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- jsonMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- jsonMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
-
- Activity activity = null;
- try {
- activity = gplusSerializer.deserialize(line);
- } catch( Exception e ) {
- LOGGER.error(line);
- e.printStackTrace();
- Assert.fail("Exception on gplus Serializer.deserialize(jsonString) : " + e);
- }
-
- try {
- String activityString = gplusSerializer.serialize(activity);
- LOGGER.debug(activityString);
- } catch (Exception e ){
- LOGGER.error(activity.toString());
- e.printStackTrace();
- Assert.fail("Exception on gplus Serializer.serialize(activity) : " + e);
- }
-
- }
- } catch( Exception e ) {
- LOGGER.error("Exception: ", e);
- Assert.fail("");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-instagram/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-instagram/pom.xml b/streams-contrib/streams-provider-gnip/gnip-edc-instagram/pom.xml
deleted file mode 100644
index 2df9797..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-instagram/pom.xml
+++ /dev/null
@@ -1,116 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-provider-gnip</artifactId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>gnip-edc-instagram</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Gnip EDC Instagram Provider</description>
-
- <dependencies>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <!--<sourceDirectory>src/main/java</sourceDirectory>-->
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jsonschema2pojo/**/*.java</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <configuration>
- <addCompileSourceRoot>true</addCompileSourceRoot>
- <generateBuilders>true</generateBuilders>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema/com/instagram/Instagram.json</sourcePath>
- </sourcePaths>
- <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>com.instagram</targetPackage>
- <useLongIntegers>true</useLongIntegers>
- <useJodaDates>true</useJodaDates>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
-
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-instagram/src/main/jsonschema/com/instagram/Instagram.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-instagram/src/main/jsonschema/com/instagram/Instagram.json b/streams-contrib/streams-provider-gnip/gnip-edc-instagram/src/main/jsonschema/com/instagram/Instagram.json
deleted file mode 100644
index 0c9e4c0..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-instagram/src/main/jsonschema/com/instagram/Instagram.json
+++ /dev/null
@@ -1,208 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "com.instagram.pojo.Instagram",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "caption": {
- "type": "object",
- "properties": {
- "created_time": {
- "format": "utc-millisec",
- "type": "string"
- },
- "from": {
- "$ref": "#/properties/user"
- },
- "id": {
- "type": "string"
- },
- "text": {
- "type": "string"
- }
- }
- },
- "comments": {
- "type": "object",
- "properties": {
- "count": {
- "type": "integer"
- },
- "data": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType" : "com.instagram.pojo.Comment",
- "properties": {
- "created_time": {
- "format": "utc-millisec",
- "type": "string"
- },
- "from": {
- "$ref": "#/properties/user"
- },
- "id": {
- "type": "string"
- },
- "text": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "created_time": {
- "format": "utc-millisec",
- "type": "string"
- },
- "filter": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "images": {
- "type": "object",
- "properties": {
- "low_resolution": {
- "type": "object",
- "javaType" : "com.instagram.pojo.Media",
- "properties": {
- "height": {
- "type": "integer"
- },
- "url": {
- "format": "uri",
- "type": "string"
- },
- "width": {
- "type": "integer"
- }
- }
- },
- "standard_resolution": {
- "$ref": "#/properties/images/properties/low_resolution"
- },
- "thumbnail": {
- "$ref": "#/properties/images/properties/low_resolution"
- }
- }
- },
- "likes": {
- "type": "object",
- "properties": {
- "count": {
- "type": "integer"
- },
- "data": {
- "type": "array",
- "items": {
- "$ref": "#/properties/user"
- }
- }
- }
- },
- "link": {
- "format": "uri",
- "type": "string"
- },
- "location": {
- "type": "object",
- "properties": {
- "latitude": {
- "type": "number"
- },
- "longitude": {
- "type": "number"
- },
- "id": {
- "type": "string"
- },
- "street_address": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "tags": {
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "type": {
- "type": "string"
- },
- "user": {
- "type": "object",
- "javaType" : "com.instagram.pojo.User",
- "properties": {
- "bio": {
- "type": "string"
- },
- "full_name": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "profile_picture": {
- "format": "uri",
- "type": "string"
- },
- "username": {
- "type": "string"
- },
- "website": {
- "format": "uri",
- "type": "string"
- }
- }
- },
- "user_has_liked": {
- "type": "boolean"
- },
- "users_in_photo": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType" : "com.instagram.pojo.UserPosition",
- "properties": {
- "user": {
- "$ref": "#/properties/user"
- },
- "position": {
- "type": "object",
- "javaType" : "com.instagram.pojo.Position",
- "properties": {
- "x": {
- "type": "number"
- },
- "y": {
- "type": "number"
- }
- }
- }
- }
- }
- },
- "videos": {
- "type": "object",
- "properties": {
- "low_resolution": {
- "$ref": "#/properties/images/properties/low_resolution"
- },
- "standard_resolution": {
- "$ref": "#/properties/images/properties/low_resolution"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-instagram/src/test/java/com/instagram/test/InstagramSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-instagram/src/test/java/com/instagram/test/InstagramSerDeTest.java b/streams-contrib/streams-provider-gnip/gnip-edc-instagram/src/test/java/com/instagram/test/InstagramSerDeTest.java
deleted file mode 100644
index 978aba6..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-instagram/src/test/java/com/instagram/test/InstagramSerDeTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.instagram.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.instagram.pojo.Instagram;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests serialization of instagram inputs
- */
-@Ignore("ignore until test resources are available.")
-public class InstagramSerDeTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(InstagramSerDeTest.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- @Test
- public void Tests()
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = InstagramSerDeTest.class.getResourceAsStream("/instagram_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- Instagram ser = mapper.readValue(line, Instagram.class);
-
- String des = mapper.writeValueAsString(ser);
- LOGGER.debug(des);
- }
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-reddit/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-reddit/pom.xml b/streams-contrib/streams-provider-gnip/gnip-edc-reddit/pom.xml
deleted file mode 100644
index 5d44c43..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-reddit/pom.xml
+++ /dev/null
@@ -1,103 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-provider-gnip</artifactId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>gnip-edc-reddit</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Gnip EDC Reddit Provider</description>
-
- <dependencies>
-
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
-
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-xml</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>jackson-xml-databind</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>gnip-powertrack</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-reddit/src/main/java/com/reddit/api/RedditActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-reddit/src/main/java/com/reddit/api/RedditActivitySerializer.java b/streams-contrib/streams-provider-gnip/gnip-edc-reddit/src/main/java/com/reddit/api/RedditActivitySerializer.java
deleted file mode 100644
index d715629..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-reddit/src/main/java/com/reddit/api/RedditActivitySerializer.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.reddit.api;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.gnip.powertrack.GnipActivityFixer;
-import org.apache.streams.pojo.json.Activity;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Created with IntelliJ IDEA.
- * User: mdelaet
- * Date: 8/29/13
- * Time: 8:32 AM
- * To change this template use File | Settings | File Templates.
- */
-public class RedditActivitySerializer implements ActivitySerializer<String> {
- private final static Logger LOGGER = LoggerFactory.getLogger(RedditActivitySerializer.class);
-
- public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
- public static final String PROVIDER_NAME = "facebook";
-
-
- @Override
- public String serializationFormat() {
- return "application/reddit+xml";
- }
-
- @Override
- public String serialize(Activity deserialized) {
- ObjectMapper jsonMapper = new ObjectMapper();
- String jsonString = new String();
- try{
- jsonString = jsonMapper.writeValueAsString(deserialized);
- }catch(Exception e){
- LOGGER.error("Exception serializing Activity Object: " + e);
- }
- return jsonString; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public Activity deserialize(String serialized) {
- ObjectMapper jsonMapper = new ObjectMapper();
- JSONObject jsonObject = new JSONObject();
- JSONObject fixedObject = new JSONObject();
-
- HashMap<String, String> raw = new HashMap<String, String>();
- raw.put("original", serialized);
-
- try{
- jsonObject = new JSONObject(serialized);
- fixedObject = GnipActivityFixer.fix(jsonObject);
- }catch(Exception e){
- LOGGER.error("Exception deserializing string: " + e);
- }
-
- Activity activity = new Activity();
- try {
- activity = jsonMapper.readValue(fixedObject.toString(), Activity.class);
- Map<String, Object> extension = ExtensionUtil.getInstance().ensureExtensions(activity);
- if (fixedObject.names().toString().contains("object")){
- if (fixedObject.getJSONObject("object").names().toString().contains("statistics")){
- if (fixedObject.getJSONObject("object").getJSONObject("statistics").names().toString().contains("upVotes")){
- extension.put("likes", fixedObject.getJSONObject("object").getJSONObject("statistics").get("upVotes"));
- }
- }
- }
-
- } catch( Exception e ) {
- LOGGER.error(jsonObject.toString());
- LOGGER.error(fixedObject.toString());
- e.printStackTrace();
- }
- return activity;
- }
-
- @Override
- public List<Activity> deserializeAll(List<String> serializedList) {
- //TODO Support
- throw new NotImplementedException("Not currently supported by this deserializer");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-reddit/src/test/java/com/reddit/api/RedditEDCAsActivityJSONTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-reddit/src/test/java/com/reddit/api/RedditEDCAsActivityJSONTest.java b/streams-contrib/streams-provider-gnip/gnip-edc-reddit/src/test/java/com/reddit/api/RedditEDCAsActivityJSONTest.java
deleted file mode 100644
index 9044e12..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-reddit/src/test/java/com/reddit/api/RedditEDCAsActivityJSONTest.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.reddit.api;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests conversion of YoutubeEDC inputs to Activity
- */
-@Ignore("ignore until test resources are available.")
-public class RedditEDCAsActivityJSONTest {
- private final static Logger LOGGER = LoggerFactory.getLogger(RedditEDCAsActivityJSONTest.class);
-
- private ObjectMapper jsonMapper;
- XmlMapper xmlMapper;
- private RedditActivitySerializer redditSerializer;
-
- public RedditEDCAsActivityJSONTest() {
- redditSerializer = new RedditActivitySerializer();
- jsonMapper = new ObjectMapper();
- xmlMapper = new XmlMapper();
- }
-
- @Test
- public void Tests() throws Exception
- {
- InputStream is = RedditEDCAsActivityJSONTest.class.getResourceAsStream("/RedditEDCFixed.json");
- if(is == null) LOGGER.debug("null");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- jsonMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- jsonMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- jsonMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
-
- Activity activity = null;
- try {
- activity = redditSerializer.deserialize(line);
- } catch( Exception e ) {
- LOGGER.error(line);
- e.printStackTrace();
- Assert.fail("Exception on redditSerializer.deserialize(jsonString) : " + e);
- }
-
- try {
- String activityString = redditSerializer.serialize(activity);
- System.out.println(jsonMapper.writeValueAsString(activity));
- }catch (Exception e ){
- LOGGER.error(activity.toString());
- e.printStackTrace();
- Assert.fail("Exception on redditSerializer.serialize(activity) : " + e);
- }
-
- //LOGGER.info(activity);
- }
- } catch( Exception e ) {
- LOGGER.error("Exception: ", e);
- Assert.fail("");
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-youtube/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-youtube/pom.xml b/streams-contrib/streams-provider-gnip/gnip-edc-youtube/pom.xml
deleted file mode 100644
index 2aace1a..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-youtube/pom.xml
+++ /dev/null
@@ -1,139 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-provider-gnip</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>gnip-edc-youtube</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Gnip EDC YouTube Provider</description>
-
- <dependencies>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>gnip-powertrack</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>aalto-xml</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <!--<sourceDirectory>src/main/java</sourceDirectory>-->
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source-jaxb2</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jaxb2</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jvnet.jaxb2.maven2</groupId>
- <artifactId>maven-jaxb2-plugin</artifactId>
- <configuration>
- <schemaDirectory>src/main/xmlschema/com/flickr/</schemaDirectory>
- <generateDirectory>target/generated-sources/jaxb2</generateDirectory>
- <verbose>true</verbose>
- <debug>true</debug>
- <encoding>${project.build.sourceEncoding}</encoding>
- <forceRegenerate>true</forceRegenerate>
- <removeOldOutput>false</removeOldOutput>
- <generatePackage>com.flickr.api</generatePackage>
- <plugins>
- <plugin>
- <groupId>org.jvnet.jaxb2_commons</groupId>
- <artifactId>jaxb2-basics</artifactId>
- <version>${jaxb2-basics.version}</version>
- </plugin>
- </plugins>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
-
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-youtube/src/test/java/com/gnip/test/YouTubeEDCSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-youtube/src/test/java/com/gnip/test/YouTubeEDCSerDeTest.java b/streams-contrib/streams-provider-gnip/gnip-edc-youtube/src/test/java/com/gnip/test/YouTubeEDCSerDeTest.java
deleted file mode 100644
index c986000..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-youtube/src/test/java/com/gnip/test/YouTubeEDCSerDeTest.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.gnip.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests serialization of YoutubeEDC inputs
- */
-@Ignore("ignore until test resources are available.")
-public class YouTubeEDCSerDeTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(YouTubeEDCSerDeTest.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-// XmlMapper mapper = new XmlMapper();
-
- @Test
- public void Tests() throws Exception
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = YouTubeEDCSerDeTest.class.getResourceAsStream("/YoutubeEDC.xml");
- if(is == null) LOGGER.debug("null");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
- XmlMapper xmlMapper = new XmlMapper();
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- ObjectMapper jsonMapper = new ObjectMapper();
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- //LOGGER.debug(line);
-
- Object activityObject = xmlMapper.readValue(line, Object.class);
-
- String jsonObject = jsonMapper.writeValueAsString(activityObject);
-
- //LOGGER.debug(jsonObject);
- }
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-youtube/src/test/java/com/gnip/test/YoutubeEDCAsActivityTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-youtube/src/test/java/com/gnip/test/YoutubeEDCAsActivityTest.java b/streams-contrib/streams-provider-gnip/gnip-edc-youtube/src/test/java/com/gnip/test/YoutubeEDCAsActivityTest.java
deleted file mode 100644
index 1c12744..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-youtube/src/test/java/com/gnip/test/YoutubeEDCAsActivityTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.gnip.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.apache.streams.gnip.powertrack.GnipActivityFixer;
-import org.apache.streams.pojo.json.Activity;
-import org.json.JSONObject;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests conversion of YoutubeEDC inputs to Activity
- */
-@Ignore("ignore until test resources are available.")
-public class YoutubeEDCAsActivityTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(YoutubeEDCAsActivityTest.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-// XmlMapper mapper = new XmlMapper();
-
- @Test @Ignore
- public void Tests() throws Exception
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = YoutubeEDCAsActivityTest.class.getResourceAsStream("/YoutubeEDC.xml");
- if(is == null) LOGGER.debug("null");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
- XmlMapper xmlMapper = new XmlMapper();
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- ObjectMapper jsonMapper = new ObjectMapper();
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- //LOGGER.debug(line);
-
- Object activityObject = xmlMapper.readValue(line, Object.class);
-
- String jsonString = jsonMapper.writeValueAsString(activityObject);
-
- JSONObject jsonObject = new JSONObject(jsonString);
-
- JSONObject fixedObject = GnipActivityFixer.fix(jsonObject);
-
- Activity activity = jsonMapper.readValue(fixedObject.toString(), Activity.class);
-
- }
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/README.md
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/README.md b/streams-contrib/streams-provider-gnip/gnip-powertrack/README.md
deleted file mode 100644
index 561db36..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:gnip-powertrack
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/pom.xml b/streams-contrib/streams-provider-gnip/gnip-powertrack/pom.xml
deleted file mode 100644
index f15fd22..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/pom.xml
+++ /dev/null
@@ -1,207 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-provider-gnip</artifactId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>gnip-powertrack</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Gnip Powertrack Provider</description>
-
- <dependencies>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-xml</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- <version>20090211</version>
- <scope>compile</scope>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jsonschema2pojo/**/*.java</source>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>add-source-jaxb2</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jaxb2</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <configuration>
- <addCompileSourceRoot>true</addCompileSourceRoot>
- <generateBuilders>true</generateBuilders>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema/com/gnip/Gnip.json</sourcePath>
- </sourcePaths>
- <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>com.gnip</targetPackage>
- <useLongIntegers>true</useLongIntegers>
- <useJodaDates>true</useJodaDates>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jvnet.jaxb2.maven2</groupId>
- <artifactId>maven-jaxb2-plugin</artifactId>
- <configuration>
- <schemaDirectory>src/main/xmlschema/com/gnip/</schemaDirectory>
- <generateDirectory>target/generated-sources/jaxb2</generateDirectory>
- <verbose>true</verbose>
- <debug>true</debug>
- <encoding>${project.build.sourceEncoding}</encoding>
- <forceRegenerate>true</forceRegenerate>
- <removeOldOutput>false</removeOldOutput>
- <generatePackage>com.gnip.api</generatePackage>
- <plugins>
- <plugin>
- <groupId>org.jvnet.jaxb2_commons</groupId>
- <artifactId>jaxb2-basics</artifactId>
- <version>${jaxb2-basics.version}</version>
- </plugin>
- </plugins>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <profiles>
- <profile>
- <id>java8</id>
- <activation>
- <jdk>[1.8,]</jdk>
- </activation>
- <build>
- <plugins>
- <plugin>
- <!-- We use this plugin to ensure that our usage of the
- maven-jaxb2-plugin is JDK 8 compatible in absence of a fix
- for https://java.net/jira/browse/MAVEN_JAXB2_PLUGIN-80. -->
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>properties-maven-plugin</artifactId>
- <version>1.0-alpha-2</version>
- <executions>
- <execution>
- <id>set-additional-system-properties</id>
- <goals>
- <goal>set-system-properties</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <properties>
- <property>
- <name>javax.xml.accessExternalSchema</name>
- <value>file,http</value>
- </property>
- </properties>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/ActivityXMLActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/ActivityXMLActivitySerializer.java b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/ActivityXMLActivitySerializer.java
deleted file mode 100644
index e75927f..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/ActivityXMLActivitySerializer.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gnip.powertrack;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.json.Activity;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Created with IntelliJ IDEA.
- * User: rebanks
- * Date: 9/5/13
- * Time: 3:07 PM
- * To change this template use File | Settings | File Templates.
- */
-public class ActivityXMLActivitySerializer implements ActivitySerializer<String> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ActivityXMLActivitySerializer.class);
-
- private ObjectMapper mapper;
- private XmlMapper xmlMapper;
-
- public ActivityXMLActivitySerializer() {
- mapper = new ObjectMapper();
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- xmlMapper = new XmlMapper();
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- }
-
-
- @Override
- public String serializationFormat() {
- return "gnip_activity_xml"; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public String serialize(Activity deserialized) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Activity deserialize(String serializedXML) {
- Activity activity = null;
- try {
- activity = xmlMapper.readValue(new StringReader(setContentIfEmpty(serializedXML)), Activity.class);
- activity = mapper.readValue(new StringReader(fixActivityXML(activity, serializedXML)), Activity.class);
- } catch (Exception e) {
- LOGGER.error("Exception correcting Gnip Activity Xml to Activity format.", e);
- LOGGER.error("Xml that caused error : {}", serializedXML);
- }
- return activity; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public List<Activity> deserializeAll(List<String> serializedList) {
- throw new NotImplementedException("Not currently supported by this deserializer");
- }
-
- private String fixActivityXML(Activity activity, String xml) throws Exception{
- JSONObject jsonObject = new JSONObject(mapper.writeValueAsString(activity));
- JSONObject newObject = (JSONObject) fixActivityXML(jsonObject);
- StringReader str = new StringReader(newObject.toString());
- newObject = formatFixedJSON(newObject);
- newObject = fixDateFormats(newObject, xml);
- newObject = createTopLevelConentField(newObject);
- return newObject.toString();
-
- }
-
- private Object fixActivityXML(Object obj) throws Exception {
- if(obj instanceof JSONObject) {
- JSONObject json = new JSONObject();
- JSONObject old = (JSONObject) obj;
- Iterator keys = old.keys();
- while(keys.hasNext()) {
- String key = (String) keys.next();
- if(StringUtils.isBlank(key)) {
- return fixActivityXML(old.get(key));
- }
- else if(!key.equals("type")){
- Object o = fixActivityXML(old.get(key));
- if(o != null)
- json.put(key, o);
- }
-
- }
- if(json.keys().hasNext())
- return json;
- else
- return null;
- }
- else {
- return obj;
- }
-
- }
-
- private JSONObject formatFixedJSON(JSONObject json) throws Exception{
- JSONObject topLink = json.optJSONObject("link");
- if(topLink != null) {
- json.put("link", topLink.getString("href"));
- }
- JSONObject actor = json.optJSONObject("actor");
- if(actor != null) {
- JSONObject link = actor.optJSONObject("link");
- if(link != null) {
- actor.put("link", link.get("href"));
- }
- }
- JSONObject object = json.optJSONObject("object");
- if(object != null) {
- JSONObject link = object.optJSONObject("link");
- if(link != null) {
- object.put("link", link.get("href"));
- }
- }
- String generator = json.optString("generator");
- if(generator != null) {
- JSONObject gen = new JSONObject();
- gen.put("displayName", generator);
- json.put("generator", gen);
- }
- return json;
- }
-
- private JSONObject fixDateFormats(JSONObject json, String xml) throws Exception{
- DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
- InputSource is = new InputSource(new StringReader(xml));
- Document doc = docBuilder.parse(is);
- //why?
- doc.getDocumentElement().normalize();
- if(json.optLong("published", -1L) != -1L) {
- json.put("published", getValueFromXML("published", doc));
- }
- if(json.optLong("updated", -1L) != -1L) {
- json.put("updated", getValueFromXML("updated", doc));
- }
- if(json.optLong("created", -1L) != -1L) {
- json.put("created", getValueFromXML("created", doc));
- }
- return json;
- }
-
- private JSONObject createTopLevelConentField(JSONObject json) throws JSONException {
- if(!json.isNull("content")) {
- return json;
- }
- JSONObject object = json.optJSONObject("object");
- if(object != null) {
- String content = object.optString("content");
- if(content == null) {
- content = object.optString("summary");
- }
- if(content != null) {
- json.put("content", content);
- }
- }
- return json;
- }
-
- private String getValueFromXML(String tag, Document doc) throws Exception{
- Element base = (Element) doc.getElementsByTagName("entry").item(0);
- return base.getElementsByTagName(tag).item(0).getTextContent();
- }
-
- private String setContentIfEmpty(String xml) throws Exception {
- DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
- InputSource is = new InputSource(new StringReader(xml));
- Document doc = docBuilder.parse(is);
- doc.getDocumentElement().normalize();
- Element base = (Element) doc.getElementsByTagName("entry").item(0);
- NodeList nodeList = base.getChildNodes();
-// for(int i=0; i < nodeList.getLength(); ++i) {
-// System.out.println(nodeList.item(i).getNodeName());
-// }
- Element obj = (Element)base.getElementsByTagName("activity:object").item(0);
- Element content = (Element)obj.getElementsByTagName("content").item(0);
-// System.out.println("Number of child nodes : "+content.getChildNodes().getLength());
-// System.out.println("Text content before : "+content.getTextContent());
- if(content.getTextContent() == null || content.getTextContent().equals("")) {
- content.setTextContent(" ");
- }
-// System.out.println("Number of child nodes after : "+content.getChildNodes().getLength());
-// System.out.println("Text content after : "+content.getTextContent());
- TransformerFactory tf = TransformerFactory.newInstance();
- Transformer transformer = tf.newTransformer();
- transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
- StringWriter writer = new StringWriter();
- transformer.transform(new DOMSource(doc), new StreamResult(writer));
- String output = writer.getBuffer().toString().replaceAll("\n|\r", "");
-// System.out.println(output);
-// System.out.println(output);
-// System.out.println(content);
- return output;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/GnipActivityFixer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/GnipActivityFixer.java b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/GnipActivityFixer.java
deleted file mode 100644
index 66ad914..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/GnipActivityFixer.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gnip.powertrack;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.apache.commons.lang.StringUtils;
-import org.apache.streams.pojo.json.Activity;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * Created with IntelliJ IDEA.
- * User: mdelaet
- * Date: 8/23/13
- * Time: 9:40 AM
- * To change this template use File | Settings | File Templates.
- */
-public class GnipActivityFixer {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GnipActivityFixer.class);
-
- private ObjectMapper mapper;
- private XmlMapper xmlMapper;
-
- public GnipActivityFixer(){
- mapper = new ObjectMapper();
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- xmlMapper = new XmlMapper();
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- };
-
- public static void findNullContents(JSONObject json, ArrayList<String> drilldownKeys, HashMap<ArrayList<String>, JSONObject> nullContents) throws Exception {
-
- Iterator itr = json.keys();
- while (itr.hasNext()){
- String element = (String) itr.next();
-
- if (StringUtils.isBlank(element)){
- nullContents.put(drilldownKeys, json);
-
- }
- else{
- try {
- drilldownKeys.add(element);
- if(json.get(element) instanceof JSONObject)
- findNullContents((JSONObject) json.get(element), drilldownKeys, nullContents);
- }catch(Exception e){
- LOGGER.info("Failed to convert in findNullKeys, " + e);
- LOGGER.error("Element : {}", element);
- LOGGER.error(json.toString());
- break;
- }
- }
- drilldownKeys = new ArrayList<String>();
- }
- }
-
- public static void editJson(JSONObject json, ArrayList<String> keyPath, Object nullFragment) throws JSONException {
- Integer numKeys = keyPath.size();
- JSONObject newJson = new JSONObject();
- if (numKeys > 1){
- for (int i = numKeys-1; i > 0; i-=1){
- String key = keyPath.get(i);
- if (i == numKeys -1){
- newJson = newJson.put(key, nullFragment);
- }
- else {
- newJson = newJson.put(key, newJson);
- }
- }
- json.put(keyPath.get(0), newJson);
- }
- else{
- json.put(keyPath.get(0), nullFragment);
- }
- }
-
- public static Activity fix(Activity activity) throws Exception {
- ObjectMapper mapper = new ObjectMapper();
-
- String des = mapper.writeValueAsString(activity);
- JSONObject json = new JSONObject(des);
-
- HashMap<ArrayList<String>, JSONObject> nullContents = new HashMap<ArrayList<String>, JSONObject>();
- ArrayList<String> drilldownKeys = new ArrayList<String>();
-
- findNullContents(json, drilldownKeys, nullContents);
-
- for ( Map.Entry<ArrayList<String>,JSONObject> entry : nullContents.entrySet() ) {
- JSONObject frag = entry.getValue();
- editJson(json, entry.getKey(), frag.get(""));
- }
-
- StringReader str = new StringReader(json.toString());
- Activity testAct = null;
- try {
- testAct = mapper.readValue(str, Activity.class);
- } catch (Exception e) {
- LOGGER.error("Exception creating activity.", e);
- LOGGER.error("JSON : {}"+json.toString());
- }
-
- return testAct;
- };
-
- public static JSONObject fix(JSONObject json) throws Exception {
-
- HashMap<ArrayList<String>, JSONObject> nullContents = new HashMap<ArrayList<String>, JSONObject>();
- ArrayList<String> drilldownKeys = new ArrayList<String>();
-
- findNullContents(json, drilldownKeys, nullContents);
-
- for ( Map.Entry<ArrayList<String>,JSONObject> entry : nullContents.entrySet() ) {
- JSONObject frag = entry.getValue();
- editJson(json, entry.getKey(), frag.get(""));
- }
-
- return json;
- };
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/PowerTrackActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/PowerTrackActivitySerializer.java b/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/PowerTrackActivitySerializer.java
deleted file mode 100644
index 55b1423..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-powertrack/src/main/java/org/apache/streams/gnip/powertrack/PowerTrackActivitySerializer.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gnip.powertrack;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.json.Activity;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.StringReader;
-import java.util.List;
-
-/**
- * Created with IntelliJ IDEA.
- * User: rebanks
- * Date: 9/5/13
- * Time: 2:29 PM
- * To change this template use File | Settings | File Templates.
- */
-public class PowerTrackActivitySerializer implements ActivitySerializer<String> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PowerTrackActivitySerializer.class);
-
- private ObjectMapper mapper;
-
- public PowerTrackActivitySerializer() {
- mapper = new ObjectMapper();
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- }
-
-
-
- @Override
- public String serializationFormat() {
- return "gnip_powertrack"; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public String serialize(Activity deserialized) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Activity deserialize(String serialized) {
- Activity activity = null;
- try {
- JSONObject jsonObject = new JSONObject(serialized);
- String content = jsonObject.optString("content", null);
- if(content == null) {
- content = jsonObject.optString("body", null);
- if(content == null) {
- content = jsonObject.optString("summary", null);
- if(content == null) {
- JSONObject object = jsonObject.optJSONObject("object");
- if(object != null) {
- content = object.optString("content", null);
- if(content == null) {
- content = object.optString("body", null);
- if(content == null) {
- content = object.optString("summary", null);
- }
- }
- }
- }
- }
-
- }
- if(content != null) {
- jsonObject.put("content", content);
- }
- String dateTime = jsonObject.optString("postedTime");
- if(dateTime != null) {
- jsonObject.put("published", dateTime);
- }
- JSONObject actor = jsonObject.optJSONObject("actor");
- if(actor != null) {
- String url = actor.optString("image");
- if(url != null) {
- JSONObject media = new JSONObject();
- media.put("url", url);
- actor.put("image", media);
- }
- }
- serialized = jsonObject.toString();
- StringReader reader = new StringReader(serialized);
- activity = this.mapper.readValue(reader, Activity.class);
- } catch (Exception e) {
- LOGGER.error("Exception deserializing powertrack string to Activity Object.", e);
- LOGGER.error("Exception on json : {}", serialized);
- }
- return activity;
- }
-
- @Override
- public List<Activity> deserializeAll(List<String> serializedList) {
- throw new NotImplementedException("Not currently supported by this deserializer");
- }
-
-}
[5/6] incubator-streams git commit: Resolves STREAMS-416
Posted by sb...@apache.org.
Resolves STREAMS-416
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/0b512d8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/0b512d8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/0b512d8b
Branch: refs/heads/master
Commit: 0b512d8b6c0d2b36a635d20b9be9a481eb67d6b9
Parents: 2c12724
Author: Elias Ponvert <ep...@gmail.com>
Authored: Mon Oct 17 10:37:51 2016 -0500
Committer: Elias Ponvert <ep...@gmail.com>
Committed: Mon Oct 17 10:37:51 2016 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 5 -
.../streams-persist-cassandra/pom.xml | 168 ----
.../configuration/CassandraConfiguration.java | 81 --
.../model/CassandraActivityStreamsEntry.java | 45 -
.../CassandraActivityStreamsRepository.java | 176 ----
.../repository/impl/CassandraKeyspace.java | 61 --
.../impl/CassandraSubscriptionRepository.java | 69 --
.../spring/streams-cassandra-context.xml | 25 -
.../CassandraActivityStreamsRepositoryTest.java | 99 ---
.../impl/CassandraActivitySubscriptionTest.java | 54 --
.../streams-provider-datasift/README.md | 18 -
.../streams-provider-datasift/pom.xml | 242 ------
.../streams/datasift/csdl/DatasiftCsdlUtil.java | 132 ---
.../DatasiftActivitySerializerProcessor.java | 97 ---
.../DatasiftTypeConverterProcessor.java | 167 ----
.../datasift/provider/DatasiftConverter.java | 38 -
.../provider/DatasiftManagedSourceSetup.java | 94 ---
.../datasift/provider/DatasiftPushProvider.java | 270 ------
.../provider/DatasiftStreamConfigurator.java | 52 --
.../provider/DatasiftStreamProvider.java | 242 ------
.../streams/datasift/provider/ErrorHandler.java | 47 --
.../streams/datasift/provider/Subscription.java | 60 --
.../serializer/DatasiftActivitySerializer.java | 65 --
.../serializer/DatasiftEventClassifier.java | 53 --
.../DatasiftInstagramActivitySerializer.java | 125 ---
.../DatasiftInteractionActivitySerializer.java | 247 ------
.../DatasiftTwitterActivitySerializer.java | 271 ------
.../datasift/util/StreamsDatasiftMapper.java | 89 --
.../org/apache/streams/datasift/Datasift.json | 462 -----------
.../streams/datasift/DatasiftConfiguration.json | 136 ----
.../datasift/DatasiftPushConfiguration.json | 20 -
.../datasift/DatasiftStreamConfiguration.json | 20 -
.../streams/datasift/DatasiftWebhookData.json | 35 -
.../datasift/facebook/DatasiftFacebook.json | 125 ---
.../datasift/instagram/DatasiftInstagram.json | 183 -----
.../interaction/DatasiftInteraction.json | 97 ---
.../datasift/twitter/DatasiftTwitter.json | 370 ---------
.../datasift/twitter/DatasiftTwitterMedia.json | 132 ---
.../datasift/twitter/DatasiftTwitterUser.json | 73 --
.../src/main/resources/datasift.conf | 25 -
.../src/site/markdown/index.md | 27 -
.../com/datasift/test/DatasiftSerDeTest.java | 75 --
.../provider/DatasiftStreamProviderTest.java | 144 ----
.../datasift/provider/ErrorHandlerTest.java | 40 -
.../datasift/provider/SubscriptionTest.java | 58 --
.../DatasiftActivitySerializerIT.java | 110 ---
.../serializer/DatasiftEventClassifierTest.java | 71 --
.../DatasiftInstagramActivitySerializerIT.java | 57 --
...DatasiftInteractionActivitySerializerIT.java | 62 --
.../DatasiftTwitterActivitySerializerIT.java | 57 --
.../gnip-edc-facebook/pom.xml | 152 ----
.../test/FacebookEDCAsActivityTest.java | 93 ---
.../facebook/test/FacebookEDCSerDeTest.java | 74 --
.../gnip-edc-flickr/pom.xml | 139 ----
.../flickr/test/FlickrEDCAsActivityTest.java | 92 ---
.../gnip/flickr/test/FlickrEDCSerDeTest.java | 76 --
.../gnip-edc-googleplus/pom.xml | 102 ---
.../com/gplus/api/GPlusActivitySerializer.java | 93 ---
.../com/gplus/api/GPlusEDCAsActivityTest.java | 95 ---
.../gnip-edc-instagram/pom.xml | 116 ---
.../jsonschema/com/instagram/Instagram.json | 208 -----
.../com/instagram/test/InstagramSerDeTest.java | 70 --
.../gnip-edc-reddit/pom.xml | 103 ---
.../reddit/api/RedditActivitySerializer.java | 107 ---
.../reddit/api/RedditEDCAsActivityJSONTest.java | 97 ---
.../gnip-edc-youtube/pom.xml | 139 ----
.../java/com/gnip/test/YouTubeEDCSerDeTest.java | 79 --
.../com/gnip/test/YoutubeEDCAsActivityTest.java | 87 --
.../gnip-powertrack/README.md | 8 -
.../gnip-powertrack/pom.xml | 207 -----
.../ActivityXMLActivitySerializer.java | 240 ------
.../gnip/powertrack/GnipActivityFixer.java | 151 ----
.../PowerTrackActivitySerializer.java | 121 ---
.../src/main/jsonschema/com/gnip/Gnip.json | 815 -------------------
.../src/main/xmlschema/com/gnip/binding.xjb | 33 -
.../src/main/xmlschema/com/gnip/entry.xsd | 398 ---------
.../gnip-powertrack/src/site/markdown/index.md | 15 -
.../test/PowerTrackDeserializationTest.java | 55 --
streams-contrib/streams-provider-gnip/pom.xml | 67 --
79 files changed, 9503 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
index 8db46c1..b50f440 100644
--- a/streams-contrib/pom.xml
+++ b/streams-contrib/pom.xml
@@ -37,7 +37,6 @@
</properties>
<modules>
- <!--<module>streams-persist-cassandra</module>-->
<module>streams-persist-console</module>
<module>streams-persist-elasticsearch</module>
<module>streams-persist-filebuffer</module>
@@ -47,16 +46,12 @@
<module>streams-persist-kafka</module>
<module>streams-persist-mongo</module>
<module>streams-amazon-aws</module>
- <!--<module>streams-processor-lucene</module>-->
- <!--<module>streams-processor-tika</module>-->
<module>streams-processor-jackson</module>
<module>streams-processor-json</module>
<module>streams-processor-urls</module>
<module>streams-processor-peoplepattern</module>
- <module>streams-provider-datasift</module>
<module>streams-provider-facebook</module>
<module>streams-provider-google</module>
- <module>streams-provider-gnip</module>
<module>streams-provider-instagram</module>
<module>streams-provider-moreover</module>
<module>streams-provider-twitter</module>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-persist-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/pom.xml b/streams-contrib/streams-persist-cassandra/pom.xml
deleted file mode 100644
index 578004c..0000000
--- a/streams-contrib/streams-persist-cassandra/pom.xml
+++ /dev/null
@@ -1,168 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-persist-cassandra</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Cassandra Module</description>
-
- <properties>
- <bundle.symbolicName>streams-persist-cassandra</bundle.symbolicName>
- <bundle.namespace>org.apache.streams</bundle.namespace>
- <easymock.version>3.2</easymock.version>
- </properties>
-
- <packaging>bundle</packaging>
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
-
- <resource>
- <directory>.</directory>
- <includes>
- <include>plugin.xml</include>
- <include>plugin.properties</include>
- <include>icons/**</include>
- </includes>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.ops4j</groupId>
- <artifactId>maven-pax-plugin</artifactId>
- <!--
- | enable improved OSGi compilation support for the bundle life-cycle.
- | to switch back to the standard bundle life-cycle, move this setting
- | down to the maven-bundle-plugin section
- -->
- <extensions>true</extensions>
- </plugin>
- <plugin>
- <groupId>org.apache.felix</groupId>
- <artifactId>maven-bundle-plugin</artifactId>
- <version>1.4.3</version>
- <!--
- | the following instructions build a simple set of public/private classes into an OSGi bundle
- -->
- <configuration>
- <instructions>
- <Bundle-SymbolicName>${bundle.symbolicName}</Bundle-SymbolicName>
- <Bundle-Version>${project.version}</Bundle-Version>
- <Export-Package>
- ${bundle.namespace};version="${project.version}",org.apache.streams.cassandra.repository.impl, org.apache.streams.cassandra.model, org.apache.streams.cassandra.configuration
- </Export-Package>
- <Private-Package>${bundle.namespace}.cassandra.repository.impl.*,${bundle.namespace}.cassandra.model, ${bundle.namespace}.cassandra.configuration </Private-Package>
- <Import-Package>
- org.apache.rave.model,org.apache.rave.portal.model.impl,
- com.datastax.driver.core, com.datastax.driver.core.exceptions, org.codehaus.jackson.map.annotate,
- javax.persistence, org.apache.commons.logging, com.google.common.collect, org.codehaus.jackson.map,
- org.apache.commons.lang,
- org.apache.streams.osgi.components.activitysubscriber,
- org.springframework.beans.factory.annotation, org.springframework.stereotype
- </Import-Package>
- </instructions>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.rave</groupId>
- <artifactId>rave-core-api</artifactId>
- <version>${rave.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.rave</groupId>
- <artifactId>rave-core</artifactId>
- <version>${rave.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.8.2</version>
- </dependency>
-
- <dependency>
- <groupId>com.datastax.cassandra</groupId>
- <artifactId>cassandra-driver-core</artifactId>
- <version>${datastax.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.2.9.Final</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams.osgi.components</groupId>
- <artifactId>activity-subscriber</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <version>${easymock.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java
deleted file mode 100644
index 195467d..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/configuration/CassandraConfiguration.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.configuration;
-
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-@Component
-public class CassandraConfiguration {
- @Value("${keyspaceName}")
- private String keyspaceName;
-
- @Value("${activitystreamsColumnFamilyName}")
- private String activitystreamsColumnFamilyName;
-
- @Value("${subscriptionColumnFamilyName}")
- private String subscriptionColumnFamilyName;
-
- @Value("${publisherColumnFamilyName}")
- private String publisherColumnFamilyName;
-
- @Value("${cassandraPort}")
- private String cassandraPort;
-
- public String getKeyspaceName() {
- return keyspaceName;
- }
-
- public void setKeyspaceName(String keyspaceName) {
- this.keyspaceName = keyspaceName;
- }
-
- public String getActivitystreamsColumnFamilyName() {
- return activitystreamsColumnFamilyName;
- }
-
- public void setActivitystreamsColumnFamilyName(String activitystreamsColumnFamilyName) {
- this.activitystreamsColumnFamilyName = activitystreamsColumnFamilyName;
- }
-
- public String getSubscriptionColumnFamilyName() {
- return subscriptionColumnFamilyName;
- }
-
- public void setSubscriptionColumnFamilyName(String subscriptionColumnFamilyName) {
- this.subscriptionColumnFamilyName = subscriptionColumnFamilyName;
- }
-
- public String getPublisherColumnFamilyName() {
- return publisherColumnFamilyName;
- }
-
- public void setPublisherColumnFamilyName(String publisherColumnFamilyName) {
- this.publisherColumnFamilyName = publisherColumnFamilyName;
- }
-
- public String getCassandraPort() {
- return cassandraPort;
- }
-
- public void setCassandraPort(String cassandraPort) {
- this.cassandraPort = cassandraPort;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java
deleted file mode 100644
index 88db8aa..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/model/CassandraActivityStreamsEntry.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.model;
-
-import org.apache.rave.model.ActivityStreamsObject;
-import org.apache.rave.portal.model.impl.ActivityStreamsEntryImpl;
-import org.apache.rave.portal.model.impl.ActivityStreamsObjectImpl;
-import org.codehaus.jackson.map.annotate.JsonDeserialize;
-
-import java.util.Date;
-
-public class CassandraActivityStreamsEntry extends ActivityStreamsEntryImpl implements Comparable<CassandraActivityStreamsEntry>{
-
- @JsonDeserialize(as=ActivityStreamsObjectImpl.class)
- private ActivityStreamsObject object;
-
- @JsonDeserialize(as=ActivityStreamsObjectImpl.class)
- private ActivityStreamsObject target;
-
- @JsonDeserialize(as=ActivityStreamsObjectImpl.class)
- private ActivityStreamsObject actor;
-
- @JsonDeserialize(as=ActivityStreamsObjectImpl.class)
- private ActivityStreamsObject provider;
-
- public int compareTo(CassandraActivityStreamsEntry entry){
- return (this.getPublished()).compareTo(entry.getPublished());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java
deleted file mode 100644
index 56e5416..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepository.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.repository.impl;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.exceptions.AlreadyExistsException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.rave.model.ActivityStreamsEntry;
-import org.apache.rave.model.ActivityStreamsObject;
-import org.apache.rave.portal.model.impl.ActivityStreamsEntryImpl;
-import org.apache.rave.portal.model.impl.ActivityStreamsObjectImpl;
-import org.apache.streams.cassandra.configuration.CassandraConfiguration;
-import org.apache.streams.cassandra.model.CassandraActivityStreamsEntry;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-
-public class CassandraActivityStreamsRepository {
-
- private static final Log LOG = LogFactory.getLog(CassandraActivityStreamsRepository.class);
-
- private CassandraKeyspace keyspace;
- private CassandraConfiguration configuration;
-
- @Autowired
- public CassandraActivityStreamsRepository(CassandraKeyspace keyspace, CassandraConfiguration configuration) {
- this.configuration = configuration;
- this.keyspace = keyspace;
-
- try {
- keyspace.getSession().execute("CREATE TABLE " + configuration.getActivitystreamsColumnFamilyName() + " (" +
- "id text, " +
- "published timestamp, " +
- "verb text, " +
- "tags text, " +
-
- "actor_displayname text, " +
- "actor_id text, " +
- "actor_url text, " +
- "actor_objecttype text, " +
-
- "target_displayname text, " +
- "target_id text, " +
- "target_url text, " +
-
- "provider_url text, " +
-
- "object_url text, " +
- "object_displayname text, " +
- "object_id text, " +
- "object_objecttype text, " +
-
- "PRIMARY KEY (id, tags, published));");
- } catch (AlreadyExistsException ignored) {
- }
- }
-
- public void save(ActivityStreamsEntry entry) {
- String sql = "INSERT INTO " + configuration.getActivitystreamsColumnFamilyName() + " (" +
- "id, published, verb, tags, " +
- "actor_displayname, actor_objecttype, actor_id, actor_url, " +
- "target_displayname, target_id, target_url, " +
- "provider_url, " +
- "object_displayname, object_objecttype, object_id, object_url) " +
- "VALUES ('" +
- entry.getId() + "','" +
- entry.getPublished().getTime() + "','" +
- entry.getVerb() + "','" +
- entry.getTags() + "','" +
-
- entry.getActor().getDisplayName() + "','" +
- entry.getActor().getObjectType() + "','" +
- entry.getActor().getId() + "','" +
- entry.getActor().getUrl() + "','" +
-
- entry.getTarget().getDisplayName() + "','" +
- entry.getTarget().getId() + "','" +
- entry.getTarget().getUrl() + "','" +
-
- entry.getProvider().getUrl() + "','" +
-
- entry.getObject().getDisplayName() + "','" +
- entry.getObject().getObjectType() + "','" +
- entry.getObject().getId() + "','" +
- entry.getObject().getUrl() +
-
- "')";
- keyspace.getSession().execute(sql);
- }
-
- public List<CassandraActivityStreamsEntry> getActivitiesForFilters(List<String> filters, Date lastUpdated) {
- List<CassandraActivityStreamsEntry> results = new ArrayList<CassandraActivityStreamsEntry>();
-
- for (String tag : filters) {
- String cql = "SELECT * FROM " + configuration.getActivitystreamsColumnFamilyName() + " WHERE ";
-
- //add filters
- cql = cql + " tags = '" + tag + "' AND ";
-
- //specify last modified
- cql = cql + "published > " + lastUpdated.getTime() + " ALLOW FILTERING";
-
- //execute the cql query and store the results
- ResultSet set = keyspace.getSession().execute(cql);
-
- //iterate through the results and create a new ActivityStreamsEntry for every result returned
-
- for (Row row : set) {
- CassandraActivityStreamsEntry entry = new CassandraActivityStreamsEntry();
- ActivityStreamsObject actor = new ActivityStreamsObjectImpl();
- ActivityStreamsObject target = new ActivityStreamsObjectImpl();
- ActivityStreamsObject object = new ActivityStreamsObjectImpl();
- ActivityStreamsObject provider = new ActivityStreamsObjectImpl();
-
- actor.setDisplayName(row.getString("actor_displayname"));
- actor.setId(row.getString("actor_id"));
- actor.setObjectType(row.getString("actor_objecttype"));
- actor.setUrl(row.getString("actor_url"));
-
- target.setDisplayName(row.getString("target_displayname"));
- target.setId(row.getString("target_id"));
- target.setUrl(row.getString("target_url"));
-
- object.setDisplayName(row.getString("object_displayname"));
- object.setObjectType(row.getString("object_objecttype"));
- object.setUrl(row.getString("object_url"));
- object.setId(row.getString("object_id"));
-
- provider.setUrl(row.getString("provider_url"));
-
- entry.setPublished(row.getDate("published"));
- entry.setVerb(row.getString("verb"));
- entry.setId(row.getString("id"));
- entry.setTags(row.getString("tags"));
- entry.setActor(actor);
- entry.setTarget(target);
- entry.setObject(object);
- entry.setProvider(provider);
-
- results.add(entry);
- }
- }
-
- return results;
- }
-
- public void dropTable(String table) {
- String cql = "DROP TABLE " + table;
- keyspace.getSession().execute(cql);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java
deleted file mode 100644
index 9106a51..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraKeyspace.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.repository.impl;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.exceptions.AlreadyExistsException;
-import org.apache.streams.cassandra.configuration.CassandraConfiguration;
-import org.springframework.beans.factory.annotation.Autowired;
-
-public class CassandraKeyspace {
- private CassandraConfiguration configuration;
- private Cluster cluster;
- private Session session;
-
- @Autowired
- public CassandraKeyspace(CassandraConfiguration configuration){
- this.configuration = configuration;
-
- cluster = Cluster.builder().addContactPoint(configuration.getCassandraPort()).build();
- session = cluster.connect();
-
- //TODO: cassandra 2 will have support for CREATE KEYSPACE IF NOT EXISTS
- try {
- session.execute("CREATE KEYSPACE " + configuration.getKeyspaceName() + " WITH replication = { 'class': 'SimpleStrategy','replication_factor' : 1 };");
- } catch (AlreadyExistsException ignored) {
- }
-
- //connect to the keyspace
- session = cluster.connect(configuration.getKeyspaceName());
- }
-
- public Session getSession(){
- return session;
- }
-
- @Override
- protected void finalize() throws Throwable {
- try {
- cluster.shutdown();
- } finally {
- super.finalize();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java
deleted file mode 100644
index f5fe471..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/repository/impl/CassandraSubscriptionRepository.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.repository.impl;
-
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.exceptions.AlreadyExistsException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.streams.cassandra.configuration.CassandraConfiguration;
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
-import org.springframework.beans.factory.annotation.Autowired;
-
-public class CassandraSubscriptionRepository {
- private static final Log LOG = LogFactory.getLog(CassandraSubscriptionRepository.class);
-
- private CassandraKeyspace keyspace;
- private CassandraConfiguration configuration;
-
- @Autowired
- public CassandraSubscriptionRepository(CassandraKeyspace keyspace, CassandraConfiguration configuration) {
- this.keyspace = keyspace;
- this.configuration = configuration;
-
- try {
- keyspace.getSession().execute("CREATE TABLE " + configuration.getSubscriptionColumnFamilyName() + " (" +
- "id text, " +
- "filters text, " +
-
- "PRIMARY KEY (id));");
- } catch (AlreadyExistsException ignored) {
- }
- }
-
- public String getFilters(String id){
- String cql = "SELECT * FROM " + configuration.getSubscriptionColumnFamilyName() + " WHERE id = '" + id+"';";
-
- ResultSet set = keyspace.getSession().execute(cql);
-
- return set.one().getString("filters");
- }
-
- public void save(ActivityStreamsSubscription subscription){
- String cql = "INSERT INTO " + configuration.getSubscriptionColumnFamilyName() + " (" +
- "id, filters) " +
- "VALUES ('" +
- subscription.getAuthToken() + "','" +
- StringUtils.join(subscription.getFilters(), " ") +
-
- "')";
- keyspace.getSession().execute(cql);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-persist-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml b/streams-contrib/streams-persist-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml
deleted file mode 100644
index 842c918..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/resources/META_INF/spring/streams-cassandra-context.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<beans
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://www.springframework.org/schema/beans"
- xmlns:context="http://www.springframework.org/schema/context"
- xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
- http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
-
-</beans>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java b/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java
deleted file mode 100644
index 978af10..0000000
--- a/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivityStreamsRepositoryTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.repository.impl;
-
-import com.datastax.driver.core.ResultSet;
-import org.apache.rave.model.ActivityStreamsEntry;
-import org.apache.rave.model.ActivityStreamsObject;
-import org.apache.rave.portal.model.impl.ActivityStreamsEntryImpl;
-import org.apache.rave.portal.model.impl.ActivityStreamsObjectImpl;
-import org.apache.streams.cassandra.configuration.CassandraConfiguration;
-import org.apache.streams.cassandra.model.CassandraActivityStreamsEntry;
-import static org.easymock.EasyMock.*;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-
-public class CassandraActivityStreamsRepositoryTest {
-
- private CassandraActivityStreamsRepository repository;
-
-
- @Before
- public void setup() {
- CassandraKeyspace keyspace = createMock(CassandraKeyspace.class);
- CassandraConfiguration configuration = createMock(CassandraConfiguration.class);
- repository = new CassandraActivityStreamsRepository(keyspace, configuration);
- }
-
- @Ignore
- @Test
- public void saveActivity() {
- ActivityStreamsEntry entry = new ActivityStreamsEntryImpl();
- ActivityStreamsObject actor = new ActivityStreamsObjectImpl();
- ActivityStreamsObject target = new ActivityStreamsObjectImpl();
- ActivityStreamsObject object = new ActivityStreamsObjectImpl();
- ActivityStreamsObject provider = new ActivityStreamsObjectImpl();
-
- actor.setId("actorid1");
- actor.setUrl("actorurl1");
- actor.setDisplayName("actorname1");
-
- target.setId("targetid1");
- target.setUrl("targeturl1");
- target.setDisplayName("r501");
-
- provider.setUrl("providerurl");
-
- object.setId("objectid1");
- object.setDisplayName("objectname1");
-
- entry.setId("dink");
- entry.setVerb("verb1");
- entry.setTags("r501");
- entry.setProvider(provider);
- Date d = new Date();
- entry.setPublished(d);
- entry.setActor(actor);
- entry.setObject(object);
- entry.setTarget(target);
-
- repository.save(entry);
- }
-
- @Ignore
- @Test
- public void getActivity() {
- String cql = "tags";
- String other = "r501";
- List<String> f = Arrays.asList(cql, other);
- Date d = new Date(0);
- List<CassandraActivityStreamsEntry> results = repository.getActivitiesForFilters(f,d);
- }
-
- @Ignore
- @Test
- public void dropTableTest(){
- repository.dropTable("coltest");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java b/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java
deleted file mode 100644
index 2a90462..0000000
--- a/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/repository/impl/CassandraActivitySubscriptionTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.repository.impl;
-
-import org.apache.streams.osgi.components.activitysubscriber.ActivityStreamsSubscription;
-import org.apache.streams.osgi.components.activitysubscriber.impl.ActivityStreamsSubscriptionImpl;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-public class CassandraActivitySubscriptionTest {
-
- public CassandraSubscriptionRepository repository;
-
-
- @Before
- public void setup() {
-// repository = new CassandraSubscriptionRepository();
- }
-
- @Ignore
- @Test
- public void saveTest(){
- ActivityStreamsSubscription subscription = new ActivityStreamsSubscriptionImpl();
- subscription.setFilters(Arrays.asList("thisis", "atest"));
- subscription.setAuthToken("subid");
-
- repository.save(subscription);
- }
-
- @Ignore
- @Test
- public void getTest(){
- String filters = repository.getFilters("subid");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/README.md
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/README.md b/streams-contrib/streams-provider-datasift/README.md
deleted file mode 100644
index a29fb16..0000000
--- a/streams-contrib/streams-provider-datasift/README.md
+++ /dev/null
@@ -1,18 +0,0 @@
-streams-provider-datasift
-=====================
-
-Datasift Provider
-
-Example configuration:
-
- datasift {
- apiKey = ""
- userName = ""
- hashes = [
- "b8aaf7cec5faa2fadbd55d651933a31e",
- "f41f054e2a2ba8d2e7b0d74f56e727d6"
- ]
- }
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml
deleted file mode 100644
index def7826..0000000
--- a/streams-contrib/streams-provider-datasift/pom.xml
+++ /dev/null
@@ -1,242 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-provider-datasift</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Datasift Provider</description>
-
- <properties>
- <skipITs>true</skipITs>
- <testDataBaseURl>http://streams.peoplepattern.com.s3.amazonaws.com/test-data/</testDataBaseURl>
- </properties>
-
-
- <dependencies>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-processor-jackson</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-provider-twitter</artifactId>
- <version>${project.version}</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-provider-instagram</artifactId>
- <version>${project.version}</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>com.datasift.client</groupId>
- <artifactId>datasift-java</artifactId>
- <version>3.2.6</version>
- <exclusions>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- <exclusion>
- <groupId>com.boundary</groupId>
- <artifactId>high-scale-lib</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.boundary</groupId>
- <artifactId>high-scale-lib</artifactId>
- <version>1.0.6</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.9.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java17</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jsonschema2pojo</source>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>add-test-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/test/java17</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <configuration>
- <addCompileSourceRoot>true</addCompileSourceRoot>
- <generateBuilders>true</generateBuilders>
- <sourceDirectory>${project.basedir}/src/main/jsonschema</sourceDirectory>
- <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>org.apache.streams.datasift</targetPackage>
- <useLongIntegers>true</useLongIntegers>
- <useJodaDates>true</useJodaDates>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.2.1</version>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>${testDataBaseURl}/${project.artifactId}.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <!--<md5>df65b5642f33676313ebe4d5b69a3fff</md5>-->
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <configuration>
- <skipTests>${skipITs}</skipTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
-
- </plugins>
-
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/csdl/DatasiftCsdlUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/csdl/DatasiftCsdlUtil.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/csdl/DatasiftCsdlUtil.java
deleted file mode 100644
index 049ed8c..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/csdl/DatasiftCsdlUtil.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.csdl;
-
-import com.google.common.collect.Lists;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.ListIterator;
-
-public class DatasiftCsdlUtil {
-
- private static final Logger log = LoggerFactory
- .getLogger(DatasiftCsdlUtil.class);
-
- public static String csdlFromTwitterUserIds(List<String> list) throws Exception {
-
- StringBuilder csdlBuilder = new StringBuilder();
-
- csdlBuilder.append("twitter.user.id in [");
- ListIterator<String> listIterator = Lists.newArrayList(list).listIterator();
- while( listIterator.hasNext() ) {
- csdlBuilder.append(listIterator.next());
- if (listIterator.hasNext())
- csdlBuilder.append(",");
- }
- csdlBuilder.append("]\n");
- csdlBuilder.append(" OR\n");
- csdlBuilder.append("twitter.in_reply_to_user_id contains_any \"");
- listIterator = Lists.newArrayList(list).listIterator();
- while( listIterator.hasNext() ) {
- csdlBuilder.append(listIterator.next());
- if (listIterator.hasNext())
- csdlBuilder.append(",");
- }
- csdlBuilder.append("\"\n");
- csdlBuilder.append(" OR\n");
- csdlBuilder.append("twitter.mention_ids in [");
- listIterator = Lists.newArrayList(list).listIterator();
- while( listIterator.hasNext() ) {
- csdlBuilder.append(listIterator.next());
- if (listIterator.hasNext())
- csdlBuilder.append(",");
- }
- csdlBuilder.append("]\n");
-
- log.debug(csdlBuilder.toString());
-
- return csdlBuilder.toString();
- }
-
- public static String csdlFromTwitterUserNames(List<String> list) throws Exception {
-
- StringBuilder csdlBuilder = new StringBuilder();
-
- csdlBuilder.append("twitter.user.screen_name contains_any \"");
- ListIterator<String> listIterator = Lists.newArrayList(list).listIterator();
- while( listIterator.hasNext() ) {
- csdlBuilder.append(listIterator.next());
- if (listIterator.hasNext())
- csdlBuilder.append(",");
- }
- csdlBuilder.append("\"\n");
- csdlBuilder.append(" OR\n");
- csdlBuilder.append("twitter.in_reply_to_screen_name contains_any \"");
- listIterator = Lists.newArrayList(list).listIterator();
- while( listIterator.hasNext() ) {
- csdlBuilder.append(listIterator.next());
- if (listIterator.hasNext())
- csdlBuilder.append(",");
- }
- csdlBuilder.append("\"\n");
- csdlBuilder.append(" OR\n");
- csdlBuilder.append("twitter.mentions contains_any \"");
- listIterator = Lists.newArrayList(list).listIterator();
- while( listIterator.hasNext() ) {
- csdlBuilder.append(listIterator.next());
- if (listIterator.hasNext())
- csdlBuilder.append(",");
- }
- csdlBuilder.append("\"\n");
-
- log.debug(csdlBuilder.toString());
-
- return csdlBuilder.toString();
- }
-
- public static String csdlFromKeywords(List<String> include, List<String> exclude) throws Exception {
-
- StringBuilder csdlBuilder = new StringBuilder();
-
- csdlBuilder.append("interaction.content contains_any \"");
- ListIterator<String> listIterator = Lists.newArrayList(include).listIterator();
- while( listIterator.hasNext() ) {
- csdlBuilder.append(listIterator.next());
- if (listIterator.hasNext())
- csdlBuilder.append(",");
- }
- csdlBuilder.append("\"\n");
- csdlBuilder.append(" AND NOT ( \n");
- csdlBuilder.append("interaction.content \"");
- listIterator = Lists.newArrayList(exclude).listIterator();
- while( listIterator.hasNext() ) {
- csdlBuilder.append(listIterator.next());
- if (listIterator.hasNext())
- csdlBuilder.append(",");
- }
- csdlBuilder.append("\"\n");
- csdlBuilder.append(")\n");
-
- log.debug(csdlBuilder.toString());
-
- return csdlBuilder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftActivitySerializerProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftActivitySerializerProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftActivitySerializerProcessor.java
deleted file mode 100644
index 43b16b2..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftActivitySerializerProcessor.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.processor;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.provider.DatasiftConverter;
-import org.apache.streams.datasift.serializer.DatasiftActivitySerializer;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- *
- */
-public class DatasiftActivitySerializerProcessor implements StreamsProcessor {
-
- private final static String STREAMS_ID = "DatasiftActivitySerializerProcessor";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftActivitySerializerProcessor.class);
-
- private ObjectMapper mapper;
- private Class outClass;
- private DatasiftActivitySerializer datasiftActivitySerializer;
-
- public final static String TERMINATE = new String("TERMINATE");
-
- public DatasiftActivitySerializerProcessor(Class outClass) {
- this.outClass = outClass;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- List<StreamsDatum> result = Lists.newLinkedList();
- Activity activity;
- try {
- Datasift node;
- if( entry.getDocument() instanceof String ) {
- node = this.mapper.readValue((String)entry.getDocument(), Datasift.class);
- } else if( entry.getDocument() instanceof Datasift ) {
- node = (Datasift) entry.getDocument();
- } else {
- node = this.mapper.convertValue(entry.getDocument(), Datasift.class);
- }
- if(node != null) {
- activity = this.datasiftActivitySerializer.deserialize(node);
- StreamsDatum datum = new StreamsDatum(activity, entry.getId(), entry.getTimestamp(), entry.getSequenceid());
- datum.setMetadata(entry.getMetadata());
- result.add(datum);
- }
- } catch (Exception e) {
- LOGGER.error("Exception converting Datasift Interaction to "+this.outClass.getName()+ " : {}", e);
- }
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.mapper = StreamsJacksonMapper.getInstance(Lists.newArrayList(StreamsDatasiftMapper.DATASIFT_FORMAT));
- this.datasiftActivitySerializer = new DatasiftActivitySerializer();
- }
-
- @Override
- public void cleanUp() {
-
- }
-
-};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
deleted file mode 100644
index dcffd1a..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.processor;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Lists;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.provider.DatasiftConverter;
-import org.apache.streams.datasift.serializer.DatasiftActivitySerializer;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.jackson.CleanAdditionalPropertiesProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- *
- */
-public class DatasiftTypeConverterProcessor implements StreamsProcessor {
-
- private final static String STREAMS_ID = "RegexUrlExtractor";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftTypeConverterProcessor.class);
-
- private ObjectMapper mapper;
- private Class outClass;
- private DatasiftActivitySerializer datasiftInteractionActivitySerializer;
- private DatasiftConverter converter;
-
- public final static String TERMINATE = new String("TERMINATE");
-
- public DatasiftTypeConverterProcessor(Class outClass) {
- this.outClass = outClass;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- List<StreamsDatum> result = Lists.newLinkedList();
- Object doc;
- try {
- if( entry.getDocument() instanceof String ) {
- ObjectNode node = this.mapper.readValue((String)entry.getDocument(), ObjectNode.class);
- doc = this.converter.convert(node, this.mapper);
- } else {
- doc = this.converter.convert(entry.getDocument(), this.mapper);
- }
- if(doc != null) {
- result.add(new StreamsDatum(doc, entry.getId()));
- }
- } catch (Exception e) {
- LOGGER.error("Exception converting Datasift Interaction to "+this.outClass.getName()+ " : {}", e);
- }
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.mapper = StreamsDatasiftMapper.getInstance();
- this.datasiftInteractionActivitySerializer = new DatasiftActivitySerializer();
- if(this.outClass.equals(Activity.class)) {
- this.converter = new ActivityConverter();
- } else if (this.outClass.equals(String.class)) {
- this.converter = new StringConverter();
- } else {
- LOGGER.warn("Using defaulting datasift converter");
- this.converter = new DefaultConverter(this.outClass);
- }
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- private class ActivityConverter implements DatasiftConverter {
-
- @Override
- public Object convert(Object toConvert, ObjectMapper mapper) {
- if(toConvert instanceof Activity)
- return toConvert;
- try {
- if(toConvert instanceof String)
- return datasiftInteractionActivitySerializer.deserialize((String) toConvert);
- return mapper.convertValue(toConvert, Activity.class);
- } catch (Exception e) {
- LOGGER.error("Exception while trying to convert {} to a Activity.", toConvert.getClass());
- LOGGER.error("Exception : {}", e);
- e.printStackTrace();
- return null;
- }
- }
-
-
- }
-
- private class StringConverter implements DatasiftConverter {
- @Override
- public Object convert(Object toConvert, ObjectMapper mapper) {
- try {
- if(toConvert instanceof String){
- return mapper.writeValueAsString(mapper.readValue((String) toConvert, Datasift.class));
- } else {
- if(toConvert.getClass().equals(Activity.class)) { //hack to remove additional properties
- ObjectNode node = mapper.convertValue(toConvert, ObjectNode.class);
- CleanAdditionalPropertiesProcessor.cleanAdditionalProperties(node);
- return mapper.writeValueAsString(node);
- } else
- return mapper.writeValueAsString(toConvert);
- }
- } catch (Exception e) {
- LOGGER.error("Exception while trying to write {} as a String.", toConvert.getClass());
- LOGGER.error("Exception : {}", e);
- return null;
- }
- }
- }
-
- private class DefaultConverter implements DatasiftConverter {
-
- private Class clazz;
-
- public DefaultConverter(Class clazz) {
- this.clazz = clazz;
- }
-
- @Override
- public Object convert(Object toConvert, ObjectMapper mapper) {
- try {
- if(toConvert instanceof String) {
- return mapper.readValue((String) toConvert, this.clazz);
- } else {
- return mapper.convertValue(toConvert, this.clazz);
- }
-
- } catch (Exception e) {
- throw new RuntimeException("Failed converting +"+ toConvert.getClass().getName()+" to "+ this.clazz.getName());
- }
- }
- }
-};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftConverter.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftConverter.java
deleted file mode 100644
index f978205..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftConverter.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.provider;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-/**
- * Converts a {@link org.apache.streams.datasift.Datasift} object to a StreamsDatum
- */
-public interface DatasiftConverter {
-
- /**
- * Converts a datasift related object to the desired resulting object.
- * @param toConvert
- * @param mapper
- * @return
- */
- public Object convert(Object toConvert, ObjectMapper mapper);
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
deleted file mode 100644
index 8200ce2..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.DataSiftClient;
-import com.datasift.client.managedsource.ManagedSource;
-import com.datasift.client.managedsource.ManagedSourceList;
-import com.datasift.client.managedsource.sources.DataSource;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Maps;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.datasift.DatasiftConfiguration;
-import org.apache.streams.datasift.managed.StreamsManagedSource;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Created by sblackmon on 8/8/14.
- */
-public class DatasiftManagedSourceSetup implements Runnable {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamProvider.class);
-
- private static DatasiftConfiguration config = DatasiftStreamConfigurator.detectConfiguration(StreamsConfigurator.config);
-
- private static final ObjectMapper MAPPER = StreamsDatasiftMapper.getInstance();
-
- DataSiftClient client;
- Map<String, ManagedSource> currentManagedSourceMap = Maps.newHashMap();
- List<StreamsManagedSource> updatedManagedSourceList;
-
- public static void main(String[] args) {
- DatasiftManagedSourceSetup job = new DatasiftManagedSourceSetup();
- (new Thread(job)).start();
- }
-
- @Override
- public void run() {
-
- setup();
-
- current();
-
- updatedManagedSourceList = config.getManagedSources();
-
- for( StreamsManagedSource source : updatedManagedSourceList ) {
- ManagedSource current = currentManagedSourceMap.get( source.getId() );
- LOGGER.info( "CURRENT: " + current );
- // merge 'em
- ManagedSource working = MAPPER.convertValue(source, ManagedSource.class);
- LOGGER.info( "WORKING: " + working );
- ManagedSource updated = client.managedSource().update(current.getName(), (DataSource) working, current).sync();
- LOGGER.info( "UPDATED: " + updated );
-
- }
-
- }
-
- public void setup() {
-
- client = new DatasiftStreamProvider(null, config).getNewClient(config.getUserName(), config.getApiKey());
- }
-
- public void current() {
- ManagedSourceList managedSources = client.managedSource().get().sync();
- Iterator<ManagedSource> managedSourceIterator = managedSources.iterator();
- while( managedSourceIterator.hasNext() ) {
- ManagedSource source = managedSourceIterator.next();
- currentManagedSourceMap.put(source.getId(), source);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftPushProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftPushProvider.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftPushProvider.java
deleted file mode 100644
index bdd2c97..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftPushProvider.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.stream.DeletedInteraction;
-import com.datasift.client.stream.StreamEventListener;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Queues;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.DatasiftConfiguration;
-import org.apache.streams.datasift.DatasiftWebhookData;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.ComponentUtils;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Resource;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import java.math.BigInteger;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.regex.Pattern;
-
-/**
- * {@code DatasiftPushProvider} is an implementation of the {@link org.apache.streams.core.StreamsProvider} interface, with
- * annotations that allow it to bind as jersey resources within streams-runtime-dropwizard.
- *
- * Whereas GenericWebhookResource outputs ObjectNode datums, DatasiftPushProvider outputs Datasift datums, with
- * metadata when the json_meta endpoint is used.
- */
-@Resource
-@Path("/streams/webhooks/datasift")
-@Produces(MediaType.APPLICATION_JSON)
-@Consumes(MediaType.APPLICATION_JSON)
-public class DatasiftPushProvider implements StreamsProvider {
-
- private final static String STREAMS_ID = "DatasiftPushProvider";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftPushProvider.class);
-
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- protected Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<>();
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- private static Pattern newLinePattern = Pattern.compile("(\\r\\n?|\\n)", Pattern.MULTILINE);
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @POST
- @Path("json")
- public Response json(@Context HttpHeaders headers,
- String body) {
-
- ObjectNode response = mapper.createObjectNode();
-
- StreamsDatum datum = new StreamsDatum(body);
-
- lock.writeLock().lock();
- ComponentUtils.offerUntilSuccess(datum, providerQueue);
- lock.writeLock().unlock();
-
- Boolean success = true;
-
- response.put("success", success);
-
- return Response.status(200).entity(response).build();
-
- }
-
- @POST
- @Path("json_new_line")
- public Response json_new_line(@Context HttpHeaders headers,
- String body) {
-
- ObjectNode response = mapper.createObjectNode();
-
- if (body.equalsIgnoreCase("{}")) {
-
- Boolean success = true;
-
- response.put("success", success);
-
- return Response.status(200).entity(response).build();
- }
-
- try {
-
- for( String item : Splitter.on(newLinePattern).split(body)) {
- StreamsDatum datum = new StreamsDatum(item);
-
- lock.writeLock().lock();
- ComponentUtils.offerUntilSuccess(datum, providerQueue);
- lock.writeLock().unlock();
-
- }
-
- Boolean success = true;
-
- response.put("success", success);
-
- return Response.status(200).entity(response).build();
-
- } catch (Exception e) {
- LOGGER.warn(e.toString(), e);
-
- Boolean success = false;
-
- response.put("success", success);
-
- return Response.status(500).entity(response).build();
-
- }
-
- }
-
- @POST
- @Path("json_meta")
- public Response json_meta(@Context HttpHeaders headers,
- String body) {
-
- //log.debug(headers.toString(), headers);
-
- //log.debug(body.toString(), body);
-
- ObjectNode response = mapper.createObjectNode();
-
- if (body.equalsIgnoreCase("{}")) {
-
- Boolean success = true;
-
- response.put("success", success);
-
- return Response.status(200).entity(response).build();
- }
-
- try {
-
- DatasiftWebhookData objectWrapper = mapper.readValue(body, DatasiftWebhookData.class);
-
- for( Datasift item : objectWrapper.getInteractions()) {
-
- String json = mapper.writeValueAsString(item);
-
- StreamsDatum datum = new StreamsDatum(json);
- if( item.getInteraction() != null &&
- !Strings.isNullOrEmpty(item.getInteraction().getId())) {
- datum.setId(item.getInteraction().getId());
- }
- if( item.getInteraction() != null &&
- item.getInteraction().getCreatedAt() != null) {
- datum.setTimestamp(item.getInteraction().getCreatedAt());
- }
- Map<String, Object> metadata = Maps.newHashMap();
- metadata.put("hash", objectWrapper.getHash());
- metadata.put("hashType", objectWrapper.getHashType());
- metadata.put("id",objectWrapper.getId());
-
- if( item.getInteraction() != null &&
- item.getInteraction().getTags() != null &&
- item.getInteraction().getTags().size() > 0) {
- metadata.put("tags", item.getInteraction().getTags());
- }
-
- datum.setMetadata(metadata);
-
- lock.writeLock().lock();
- ComponentUtils.offerUntilSuccess(datum, providerQueue);
- lock.writeLock().unlock();
- }
-
- Boolean success = true;
-
- response.put("success", success);
-
- return Response.status(200).entity(response).build();
-
- } catch (Exception e) {
- LOGGER.warn(e.toString(), e);
- }
-
- return Response.status(500).build();
- }
-
- @Override
- public void startStream() {
- return;
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- lock.writeLock().lock();
- current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(providerQueue));
- providerQueue.clear();
- lock.writeLock().unlock();
-
- return current;
-
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return true;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
deleted file mode 100644
index 6ec395d..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.provider;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
-import org.apache.streams.datasift.DatasiftConfiguration;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class DatasiftStreamConfigurator {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamConfigurator.class);
-
- private static final ObjectMapper MAPPER = StreamsDatasiftMapper.getInstance();
-
- public static DatasiftConfiguration detectConfiguration(Config datasift) {
-
- DatasiftConfiguration datasiftConfiguration = null;
-
- try {
- datasiftConfiguration = MAPPER.readValue(datasift.root().render(ConfigRenderOptions.concise()), DatasiftConfiguration.class);
- } catch (Exception e) {
- e.printStackTrace();
- LOGGER.warn("Could not parse datasiftConfiguration");
- }
- return datasiftConfiguration;
- }
-
-}
[3/6] incubator-streams git commit: Resolves STREAMS-416
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/interaction/DatasiftInteraction.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/interaction/DatasiftInteraction.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/interaction/DatasiftInteraction.json
deleted file mode 100644
index 5ff366a..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/interaction/DatasiftInteraction.json
+++ /dev/null
@@ -1,97 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.datasift.interaction.Interaction",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "author": {
- "javaType": "org.apache.streams.datasift.interaction.Author",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "avatar": {
- "type": "string"
- },
- "id": {
- "type": "integer"
- },
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "username": {
- "type": "string"
- }
- }
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "geo": {
- "javaType": "org.apache.streams.datasift.interaction.Geo",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "latitude": {
- "type": "number"
- },
- "longitude": {
- "type": "number"
- }
- }
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "received_at": {
- "type": "long"
- },
- "subtype": {
- "type": "string"
- },
- "schema": {
- "dynamic": "true",
- "properties": {
- "version": {
- "type": "integer"
- }
- }
- },
- "source": {
- "type": "string"
- },
- "subtype": {
- "type": "string"
- },
- "tags": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
deleted file mode 100644
index 8c373a3..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitter.json
+++ /dev/null
@@ -1,370 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.twitter.Twitter",
- "javaInterfaces": ["java.io.Serializable"],
- "properties" :
- {
- "created_at" :
- {
- "type" : "string"
- },
- "display_urls" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "domains" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "filter_level" :
- {
- "type" : "string"
- },
- "geo" :
- {
- "javaType" : "org.apache.streams.datasift.twitter.Geo",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "latitude" :
- {
- "type" : "number"
- },
- "longitude" :
- {
- "type" : "number"
- }
- }
- },
- "hashtags" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "id" :
- {
- "type" : "string"
- },
- "in_reply_to_screen_name" :
- {
- "type" : "string"
- },
- "in_reply_to_status_id" :
- {
- "type" : "string"
- },
- "in_reply_to_user_id" :
- {
- "type" : "string"
- },
- "lang" :
- {
- "type" : "string"
- },
- "links" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "media" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "object",
- "$ref" : "DatasiftTwitterMedia.json"
- }
- ]
- },
- "mention_ids" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "mentions" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "place" :
- {
- "javaType" : "org.apache.streams.datasift.twitter.Place",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "attributes" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "street_address" :
- {
- "type" : "string"
- }
- }
- },
- "country" :
- {
- "type" : "string"
- },
- "country_code" :
- {
- "type" : "string"
- },
- "full_name" :
- {
- "type" : "string"
- },
- "id" :
- {
- "type" : "string"
- },
- "name" :
- {
- "type" : "string"
- },
- "place_type" :
- {
- "type" : "string"
- },
- "url" :
- {
- "type" : "string"
- }
- }
- },
- "retweet" :
- {
- "javaType" : "org.apache.streams.datasift.twitter.Retweet",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "count" :
- {
- "type" : "integer"
- },
- "created_at" :
- {
- "type" : "string"
- },
- "domains" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "hashtags" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "id" :
- {
- "type" : "string"
- },
- "lang" :
- {
- "type" : "string"
- },
- "links" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "media" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "object",
- "$ref" : "DatasiftTwitterMedia.json"
- }
- ]
- },
- "mention_ids" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "integer"
- }
- ]
- },
- "mentions" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "source" :
- {
- "type" : "string"
- },
- "text" :
- {
- "type" : "string"
- },
- "user" :
- {
- "type": "object",
- "$ref": "DatasiftTwitterUser.json"
- }
- }
- },
- "retweeted" :
- {
- "javaType" : "org.apache.streams.datasift.twitter.Retweeted",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "created_at" :
- {
- "type" : "string"
- },
- "geo" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "latitude" :
- {
- "type" : "number"
- },
- "longitude" :
- {
- "type" : "number"
- }
- }
- },
- "id" :
- {
- "type" : "string"
- },
- "place" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "attributes" :
- {
- "type" : "object",
- "dynamic" : "true"
- },
- "country" :
- {
- "type" : "string"
- },
- "country_code" :
- {
- "type" : "string"
- },
- "full_name" :
- {
- "type" : "string"
- },
- "id" :
- {
- "type" : "string"
- },
- "name" :
- {
- "type" : "string"
- },
- "place_type" :
- {
- "type" : "string"
- },
- "url" :
- {
- "type" : "string"
- }
- }
- },
- "source" :
- {
- "type" : "string"
- },
- "user" :
- {
- "type": "object",
- "$ref": "DatasiftTwitterUser.json"
- }
- }
- },
- "source" :
- {
- "type" : "string"
- },
- "text" :
- {
- "type" : "string"
- },
- "user" :
- {
- "type": "object",
- "$ref": "DatasiftTwitterUser.json"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterMedia.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterMedia.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterMedia.json
deleted file mode 100644
index ee19320..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterMedia.json
+++ /dev/null
@@ -1,132 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.twitter.Media",
- "javaInterfaces": ["java.io.Serializable"],
- "properties" :
- {
- "display_url" :
- {
- "type" : "string"
- },
- "expanded_url" :
- {
- "type" : "string"
- },
- "id" :
- {
- "type" : "integer"
- },
- "id_str" :
- {
- "type" : "string"
- },
- "media_url" :
- {
- "type" : "string"
- },
- "media_url_https" :
- {
- "type" : "string"
- },
- "sizes" :
- {
- "properties" :
- {
- "large" :
- {
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- },
- "medium" :
- {
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- },
- "small" :
- {
- "dynamic" : "true",
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- },
- "thumb" :
- {
- "properties" :
- {
- "h" :
- {
- "type" : "integer"
- },
- "resize" :
- {
- "type" : "string"
- },
- "w" :
- {
- "type" : "integer"
- }
- }
- }
- }
- },
- "source_status_id" :
- {
- "type" : "integer"
- },
- "source_status_id_str" :
- {
- "type" : "string"
- },
- "type" :
- {
- "type" : "string"
- },
- "url" :
- {
- "type" : "string"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterUser.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterUser.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterUser.json
deleted file mode 100644
index d444ee0..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/twitter/DatasiftTwitterUser.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.datasift.twitter.DatasiftTwitterUser",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "description": {
- "type": "string"
- },
- "favourites_count": {
- "type": "integer"
- },
- "followers_count": {
- "type": "integer"
- },
- "friends_count": {
- "type": "integer"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "id": {
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "listed_count": {
- "type": "integer"
- },
- "location": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "profile_image_url": {
- "type": "string"
- },
- "profile_image_url_https": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "statuses_count": {
- "type": "integer"
- },
- "time_zone": {
- "type": "string"
- },
- "url": {
- "type": "string"
- },
- "utc_offset": {
- "type": "integer"
- },
- "verified": {
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/resources/datasift.conf
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/resources/datasift.conf b/streams-contrib/streams-provider-datasift/src/main/resources/datasift.conf
deleted file mode 100644
index bfe6125..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/resources/datasift.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-datasift {
- apiKey = ""
- userName = ""
- hashes = [
- "b8aaf7cec5faa2fadbd55d651933a31e",
- "f41f054e2a2ba8d2e7b0d74f56e727d6"
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/site/markdown/index.md b/streams-contrib/streams-provider-datasift/src/site/markdown/index.md
deleted file mode 100644
index 87b9dee..0000000
--- a/streams-contrib/streams-provider-datasift/src/site/markdown/index.md
+++ /dev/null
@@ -1,27 +0,0 @@
-streams-provider-datasift
-==============
-
-streams-provider-datasift contains schema definitions, providers, conversions, and utility classes.
-
-## Data Types
-
-| Schema |
-|--------|
-| [Datasift.json](org/apache/streams/datasift/Datasift.json "Datasift.json")
-| [DatasiftInteraction.json](org/apache/streams/datasift/interaction/DatasiftInteraction.json "DatasiftInteraction.json")
-| [DatasiftFacebook.json](org/apache/streams/datasift/facebook/DatasiftFacebook.json "DatasiftFacebook.json")
-| [DatasiftInstagram.json](org/apache/streams/datasift/instagram/DatasiftInstagram.json "DatasiftInstagram.json")
-| [DatasiftWebhookData.json](org/apache/streams/datasift/DatasiftWebhookData.json "DatasiftWebhookData.json")
-
-## Configuration
-
-| Schema |
-|--------|
-| [DatasiftConfiguration.json](org/apache/streams/datasift/DatasiftConfiguration.json "DatasiftConfiguration.json") [DatasiftConfiguration.html](apidocs/org/apache/streams/datasift/DatasiftConfiguration.html "javadoc") |
-| [DatasiftPushConfiguration.json](org/apache/streams/datasift/DatasiftPushConfiguration.json "DatasiftPushConfiguration.json") [DatasiftPushConfiguration.html](apidocs/org/apache/streams/datasift/DatasiftPushConfiguration.html "javadoc") |
-| [DatasiftStreamConfiguration.json](org/apache/streams/datasift/DatasiftStreamConfiguration.json "DatasiftStreamConfiguration.json") [DatasiftStreamConfiguration.html](apidocs/org/apache/streams/datasift/DatasiftStreamConfiguration.html "javadoc") |
-| [DatasiftWebhookConfiguration.json](org/apache/streams/datasift/DatasiftWebhookConfiguration.json "DatasiftWebhookConfiguration.json") [DatasiftWebhookConfiguration.html](apidocs/org/apache/streams/datasift/DatasiftWebhookConfiguration.html "javadoc") |
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/test/java17/com/datasift/test/DatasiftSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/com/datasift/test/DatasiftSerDeTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/com/datasift/test/DatasiftSerDeTest.java
deleted file mode 100644
index 11a384d..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/java17/com/datasift/test/DatasiftSerDeTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.datasift.test;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests serialization of Datasift inputs
- */
-@Ignore("ignore until test resources are available.")
-public class DatasiftSerDeTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftSerDeTest.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance(Lists.newArrayList(StreamsDatasiftMapper.DATASIFT_FORMAT));
-
- @Test
- public void Tests()
- {
- InputStream is = DatasiftSerDeTest.class.getResourceAsStream("/part-r-00000.json");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- Datasift ser = mapper.readValue(line, Datasift.class);
-
- String de = mapper.writeValueAsString(ser);
-
- LOGGER.debug(de);
-
- Datasift serde = mapper.readValue(de, Datasift.class);
-
- Assert.assertEquals(ser, serde);
-
- LOGGER.debug(mapper.writeValueAsString(serde));
- }
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftStreamProviderTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftStreamProviderTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftStreamProviderTest.java
deleted file mode 100644
index a7d14b4..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/DatasiftStreamProviderTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.DataSiftClient;
-import com.datasift.client.stream.ConnectionManager;
-import com.datasift.client.stream.StreamEventListener;
-import com.google.common.collect.Lists;
-import org.apache.streams.datasift.DatasiftConfiguration;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.times;
-
-/**
- * Requires Java version 1.7!
- */
-public class DatasiftStreamProviderTest {
-
- private static final String HASH1 = "fake1";
- private static final String HASH2 = "fake2";
- private static final String HASH3 = "fake3";
-
- @Test
- public void startStreamForHash() {
- DatasiftStreamProvider.DeleteHandler handler = new DatasiftStreamProvider.DeleteHandler();
- final List<DataSiftClient> clientList = Lists.newLinkedList();
- DatasiftStreamProvider provider = createStubbedProvider(clientList, handler);
- provider.prepare(null);
- provider.startStreamForHash(HASH1);
- assertEquals(1, clientList.size());
- provider.startStreamForHash(HASH2);
- assertEquals(2, clientList.size());
- provider.startStreamForHash(HASH3);
- }
-
- @Test
- public void testStartStream() {
- DatasiftStreamProvider.DeleteHandler handler = new DatasiftStreamProvider.DeleteHandler();
- final List<DataSiftClient> clientList = Lists.newLinkedList();
- DatasiftStreamProvider provider = createStubbedProvider(clientList, handler);
- provider.prepare(null);
- provider.startStream();
- assertEquals(3, clientList.size());
- }
-
- @Test
- public void testShutDownStream() {
- DatasiftStreamProvider.DeleteHandler handler = new DatasiftStreamProvider.DeleteHandler();
- final List<DataSiftClient> clientList = Lists.newLinkedList();
- DatasiftStreamProvider provider = createStubbedProvider(clientList, handler);
- provider.prepare(null);
- provider.startStream();
- assertEquals(3, clientList.size());
- int shutDownCount = 0;
- DataSiftClient client = clientList.get(0);
- provider.shutDownStream(HASH1);
- Mockito.verify(client, times(1)).shutdown();
- client = clientList.get(1);
- Mockito.verify(client, times(0)).shutdown();
- client = clientList.get(2);
- Mockito.verify(client, times(0)).shutdown();
-
- provider.shutDownStream(HASH3);
- Mockito.verify(client, times(1)).shutdown();
- client = clientList.get(1);
- Mockito.verify(client, times(0)).shutdown();
- client = clientList.get(2);
- Mockito.verify(client, times(1)).shutdown();
- }
-
- @Test
- public void testStartAlreadyInprogressStream() {
- DatasiftStreamProvider.DeleteHandler handler = new DatasiftStreamProvider.DeleteHandler();
- final List<DataSiftClient> clientList = Lists.newLinkedList();
- DatasiftStreamProvider provider = createStubbedProvider(clientList, handler);
- provider.prepare(null);
- provider.startStream();
- assertEquals(3, clientList.size());
- int shutDownCount = 0;
- DataSiftClient client = clientList.get(0);
- provider.startStreamForHash(HASH1);
- assertEquals(4, clientList.size());
- Mockito.verify(client, times(1)).shutdown();
- client = clientList.get(1);
- Mockito.verify(client, times(0)).shutdown();
- client = clientList.get(2);
- Mockito.verify(client, times(0)).shutdown();
- client = clientList.get(3);
- Mockito.verify(client, times(0)).shutdown();
- }
-
-
-
-
- private DatasiftStreamProvider createStubbedProvider(final List<DataSiftClient> clientList, StreamEventListener listener) {
- DatasiftStreamProvider provider = new DatasiftStreamProvider(listener, getTestConfiguration()) {
- @Override
- protected DataSiftClient getNewClient(String userName, String apiKey) {
- DataSiftClient client = Mockito.mock(DataSiftClient.class);
- ConnectionManager mockConnection = Mockito.mock(ConnectionManager.class);
- Mockito.when(client.liveStream()).thenReturn(mockConnection);
- clientList.add(client);
- return client;
- }
- };
- return provider;
- }
-
- private DatasiftConfiguration getTestConfiguration() {
- DatasiftConfiguration config = new DatasiftConfiguration();
- config.setUserName("fakeName");
- config.setApiKey("fakeApiKey");
- List<String> streamHashes = Lists.newLinkedList();
- streamHashes.add(HASH1);
- streamHashes.add(HASH2);
- streamHashes.add(HASH3);
- config.setStreamHash(streamHashes);
- return config;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/ErrorHandlerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/ErrorHandlerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/ErrorHandlerTest.java
deleted file mode 100644
index 2cd411a..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/ErrorHandlerTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.provider;
-
-import org.junit.Test;
-import org.mockito.Mockito;
-
-/**
- * Requires Java version 1.7!
- */
-public class ErrorHandlerTest {
-
- @Test
- public void testErrorHandler() {
- DatasiftStreamProvider mockProvider = Mockito.mock(DatasiftStreamProvider.class);
- String streamHash = "fakeHash1";
- ErrorHandler handler = new ErrorHandler(mockProvider, streamHash);
- handler.exceptionCaught(new Exception("TEST EXCEPTION"));
- Mockito.verify(mockProvider).startStreamForHash(streamHash);
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/SubscriptionTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/SubscriptionTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/SubscriptionTest.java
deleted file mode 100644
index f194792..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/provider/SubscriptionTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.core.Stream;
-import com.datasift.client.stream.Interaction;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Requires Java version 1.7!
- */
-public class SubscriptionTest {
-
- @Test
- public void testSubscriptionOnMessage() {
- Stream mockStream = Mockito.mock(Stream.class);
- Mockito.when(mockStream.hash()).thenReturn("1");
- Queue<Interaction> interactionQueue = new ConcurrentLinkedQueue<Interaction>();
- Subscription subscriptionStub = new Subscription(mockStream, interactionQueue);
- addInteractions(1, subscriptionStub);
- assertEquals(1, interactionQueue.size());
- addInteractions(30, subscriptionStub);
- assertEquals(31, interactionQueue.size());
- while(!interactionQueue.isEmpty())
- interactionQueue.poll();
- addInteractions(5, subscriptionStub);
- assertEquals(5, interactionQueue.size());
- }
-
- private void addInteractions(int numToAdd, Subscription subscription) {
- for(int i=0; i < numToAdd; ++i) {
- subscription.onMessage(Mockito.mock(Interaction.class));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerIT.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerIT.java
deleted file mode 100644
index 69c49fd..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerIT.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.util.files.StreamsScannerUtil;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Scanner;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Tests serialization and conversion of Datasift inputs
- */
-public class DatasiftActivitySerializerIT {
-
- protected ActivitySerializer SERIALIZER;
-
- protected static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(Lists.newArrayList(StreamsDatasiftMapper.DATASIFT_FORMAT));
-
- @Before
- public void initSerializer() {
- SERIALIZER = new DatasiftActivitySerializer();
- }
-
- @Test
- public void testConversion() throws Exception {
-
- Scanner scanner = StreamsScannerUtil.getInstance("/rand_sample_datasift_json.txt");
-
- String line = null;
- while(scanner.hasNextLine()) {
- try {
- line = scanner.nextLine();
- Datasift item = MAPPER.readValue(line, Datasift.class);
- testConversion(item);
- String json = MAPPER.writeValueAsString(item);
- testDeserNoNull(json);
- testDeserNoAddProps(json);
- } catch (Exception e) {
- System.err.println(line);
- throw e;
- }
- }
- }
-
- /**
- * Test that the minimum number of things that an activity has
- * @param item
- */
- protected void testConversion(Datasift item) throws Exception {
- Activity activity = SERIALIZER.deserialize(item);
- assertNotNull("activity.id", activity.getId());
- assertNotNull("activity.published", activity.getPublished());
- assertNotNull("activity.provider", activity.getProvider());
- assertNotNull("activity.url", activity.getUrl());
- assertNotNull("activity.verb", activity.getVerb());
- Actor actor = activity.getActor();
- assertNotNull("activity.actor", actor);
- }
-
- /**
- * Test that null fields are not present
- * @param json
- */
- protected void testDeserNoNull(String json) throws Exception {
- int nulls = StringUtils.countMatches(json, ":null");
- assertEquals(0l, (long)nulls);
-
- }
-
- /**
- * Test that null fields are not present
- * @param json
- */
- protected void testDeserNoAddProps(String json) throws Exception {
- int nulls = StringUtils.countMatches(json, "additionalProperties:{");
- assertEquals(0l, (long)nulls);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java
deleted file mode 100644
index 257e8ef..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.instagram.Instagram;
-import org.apache.streams.datasift.twitter.Twitter;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.files.StreamsScannerUtil;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Scanner;
-
-/**
- * Tests serialization and conversion of Datasift inputs
- */
-@Ignore("ignore until test resources are available.")
-public class DatasiftEventClassifierTest {
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(Lists.newArrayList(StreamsDatasiftMapper.DATASIFT_FORMAT));
-
- @Test
- public void testTwitterDetection() throws Exception {
-
- Scanner scanner = StreamsScannerUtil.getInstance("/twitter_datasift_json.txt");
-
- String line = null;
- while(scanner.hasNextLine()) {
- line = scanner.nextLine();
- Datasift datasift = MAPPER.readValue(line, Datasift.class);
- assert(DatasiftEventClassifier.detectClass(datasift) == Twitter.class);
- assert(DatasiftEventClassifier.bestSerializer(datasift) instanceof DatasiftTwitterActivitySerializer);
- }
- }
-
- @Test
- public void testInstagramDetection() throws Exception {
-
- Scanner scanner = StreamsScannerUtil.getInstance("/instagram_datasift_json.txt");
-
- String line = null;
- while(scanner.hasNextLine()) {
- line = scanner.nextLine();
- Datasift datasift = MAPPER.readValue(line, Datasift.class);
- assert(DatasiftEventClassifier.detectClass(datasift) == Instagram.class);
- assert(DatasiftEventClassifier.bestSerializer(datasift) instanceof DatasiftInstagramActivitySerializer);
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerIT.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerIT.java
deleted file mode 100644
index aa57454..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerIT.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.util.files.StreamsScannerUtil;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Scanner;
-
-/**
- * Tests serialization and conversion of Datasift inputs
- */
-public class DatasiftInstagramActivitySerializerIT extends DatasiftActivitySerializerIT {
-
- @Before
- @Override
- public void initSerializer() {
- SERIALIZER = new DatasiftInstagramActivitySerializer();
- }
-
- @Test
- @Override
- public void testConversion() throws Exception {
-
- Scanner scanner = StreamsScannerUtil.getInstance("/instagram_datasift_json.txt");
-
- String line = null;
- while(scanner.hasNextLine()) {
- line = scanner.nextLine();
- Datasift item = MAPPER.readValue(line, Datasift.class);
- testConversion(item);
- String json = MAPPER.writeValueAsString(item);
- testDeserNoNull(json);
- testDeserNoAddProps(json);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerIT.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerIT.java
deleted file mode 100644
index 40a1e95..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerIT.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.util.files.StreamsScannerUtil;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Scanner;
-
-/**
- * Tests serialization and conversion of Datasift inputs
- */
-public class DatasiftInteractionActivitySerializerIT extends DatasiftActivitySerializerIT {
-
- @Before
- @Override
- public void initSerializer() {
- SERIALIZER = new DatasiftInteractionActivitySerializer();
- }
-
- @Test
- @Override
- public void testConversion() throws Exception {
-
- Scanner scanner = StreamsScannerUtil.getInstance("/rand_sample_datasift_json.txt");
-
- String line = null;
- while(scanner.hasNextLine()) {
- try {
- line = scanner.nextLine();
- Datasift item = MAPPER.readValue(line, Datasift.class);
- testConversion(item);
- String json = MAPPER.writeValueAsString(item);
- testDeserNoNull(json);
- testDeserNoAddProps(json);
- } catch (Exception e) {
- System.err.println(line);
- throw e;
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerIT.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerIT.java
deleted file mode 100644
index 42241f1..0000000
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerIT.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.util.files.StreamsScannerUtil;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Scanner;
-
-/**
- * Tests serialization and conversion of Datasift inputs
- */
-public class DatasiftTwitterActivitySerializerIT extends DatasiftActivitySerializerIT {
-
- @Before
- @Override
- public void initSerializer() {
- SERIALIZER = new DatasiftTwitterActivitySerializer();
- }
-
- @Test
- @Override
- public void testConversion() throws Exception {
-
- Scanner scanner = StreamsScannerUtil.getInstance("/twitter_datasift_json.txt");
-
- String line = null;
- while(scanner.hasNextLine()) {
- line = scanner.nextLine();
- Datasift item = MAPPER.readValue(line, Datasift.class);
- testConversion(item);
- String json = MAPPER.writeValueAsString(item);
- testDeserNoNull(json);
- testDeserNoAddProps(json);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-facebook/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-facebook/pom.xml b/streams-contrib/streams-provider-gnip/gnip-edc-facebook/pom.xml
deleted file mode 100644
index ebbd18a..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-facebook/pom.xml
+++ /dev/null
@@ -1,152 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>streams-provider-gnip</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>gnip-edc-facebook</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Gnip EDC Facebook Provider</description>
-
- <dependencies>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-joda</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-provider-facebook</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>gnip-powertrack</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>aalto-xml</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source-jaxb2</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jaxb2</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jvnet.jaxb2.maven2</groupId>
- <artifactId>maven-jaxb2-plugin</artifactId>
- <configuration>
- <schemaDirectory>src/main/xmlschema/com/facebook/</schemaDirectory>
- <generateDirectory>target/generated-sources/jaxb2</generateDirectory>
- <verbose>true</verbose>
- <debug>true</debug>
- <encoding>${project.build.sourceEncoding}</encoding>
- <forceRegenerate>true</forceRegenerate>
- <removeOldOutput>false</removeOldOutput>
- <generatePackage>org.apache.streams.facebook.api</generatePackage>
- <plugins>
- <plugin>
- <groupId>org.jvnet.jaxb2_commons</groupId>
- <artifactId>jaxb2-basics</artifactId>
- <version>${jaxb2-basics.version}</version>
- </plugin>
- </plugins>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
-
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-facebook/src/test/java/org/apache/streams/gnip/facebook/test/FacebookEDCAsActivityTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-facebook/src/test/java/org/apache/streams/gnip/facebook/test/FacebookEDCAsActivityTest.java b/streams-contrib/streams-provider-gnip/gnip-edc-facebook/src/test/java/org/apache/streams/gnip/facebook/test/FacebookEDCAsActivityTest.java
deleted file mode 100644
index 18fbfac..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-facebook/src/test/java/org/apache/streams/gnip/facebook/test/FacebookEDCAsActivityTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gnip.facebook.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.apache.commons.lang.StringUtils;
-import org.apache.streams.gnip.powertrack.GnipActivityFixer;
-import org.apache.streams.pojo.json.Activity;
-import org.json.JSONObject;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests conversion of FacebookEDC inputs to Activity
- */
-@Ignore("ignore until test resources are available.")
-public class FacebookEDCAsActivityTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(FacebookEDCAsActivityTest.class);
-
- private ObjectMapper jsonMapper = new ObjectMapper();
- XmlMapper xmlMapper = new XmlMapper();
-
- @Test
- public void Tests() throws Exception
- {
- InputStream is = FacebookEDCAsActivityTest.class.getResourceAsStream("/FacebookEDC.xml");
- if(is == null) LOGGER.debug("null");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- jsonMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- jsonMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- jsonMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if(!StringUtils.isEmpty(line))
- {
- LOGGER.debug(line);
- Object activityObject = xmlMapper.readValue(line, Object.class);
-
- String jsonString = jsonMapper.writeValueAsString(activityObject);
-
- JSONObject jsonObject = new JSONObject(jsonString);
-
- JSONObject fixedObject = GnipActivityFixer.fix(jsonObject);
-
- Activity activity = null;
- try {
- activity = jsonMapper.readValue(fixedObject.toString(), Activity.class);
- } catch( Exception e ) {
- LOGGER.error(jsonObject.toString());
- LOGGER.error(fixedObject.toString());
- e.printStackTrace();
- Assert.fail();
- }
- }
- }
- } catch( Exception e ) {
- LOGGER.error(e.getMessage());
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-facebook/src/test/java/org/apache/streams/gnip/facebook/test/FacebookEDCSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-facebook/src/test/java/org/apache/streams/gnip/facebook/test/FacebookEDCSerDeTest.java b/streams-contrib/streams-provider-gnip/gnip-edc-facebook/src/test/java/org/apache/streams/gnip/facebook/test/FacebookEDCSerDeTest.java
deleted file mode 100644
index 91a7634..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-facebook/src/test/java/org/apache/streams/gnip/facebook/test/FacebookEDCSerDeTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gnip.facebook.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import com.gnip.api.Entry;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests serialization of FacebookEDC inputs
- */
-@Ignore("ignore until test resources are available.")
-public class FacebookEDCSerDeTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(FacebookEDCSerDeTest.class);
-
- XmlMapper xmlMapper = new XmlMapper();
-
- @Test
- public void Tests() throws Exception
- {
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = FacebookEDCSerDeTest.class.getResourceAsStream("/FacebookEDC.xml");
- if(is == null) LOGGER.debug("null");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- Entry xmlObject = xmlMapper.readValue(line, Entry.class);
-
- String xml = xmlMapper.writeValueAsString(xmlObject);
-
- }
- } catch( Exception e ) {
- LOGGER.error(e.getMessage());
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-flickr/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-flickr/pom.xml b/streams-contrib/streams-provider-gnip/gnip-edc-flickr/pom.xml
deleted file mode 100644
index 3ebd84d..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-flickr/pom.xml
+++ /dev/null
@@ -1,139 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>streams-provider-gnip</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>gnip-edc-flickr</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Gnip EDC Flickr Provider</description>
-
- <dependencies>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>gnip-powertrack</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>aalto-xml</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source-jaxb2</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jaxb2</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jvnet.jaxb2.maven2</groupId>
- <artifactId>maven-jaxb2-plugin</artifactId>
- <configuration>
- <schemaDirectory>src/main/xmlschema/com/flickr/</schemaDirectory>
- <generateDirectory>target/generated-sources/jaxb2</generateDirectory>
- <verbose>true</verbose>
- <debug>true</debug>
- <encoding>${project.build.sourceEncoding}</encoding>
- <forceRegenerate>true</forceRegenerate>
- <removeOldOutput>false</removeOldOutput>
- <generatePackage>com.flickr.api</generatePackage>
- <plugins>
- <plugin>
- <groupId>org.jvnet.jaxb2_commons</groupId>
- <artifactId>jaxb2-basics</artifactId>
- <version>${jaxb2-basics.version}</version>
- </plugin>
- </plugins>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
-
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-flickr/src/test/java/org/apache/streams/gnip/flickr/test/FlickrEDCAsActivityTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-flickr/src/test/java/org/apache/streams/gnip/flickr/test/FlickrEDCAsActivityTest.java b/streams-contrib/streams-provider-gnip/gnip-edc-flickr/src/test/java/org/apache/streams/gnip/flickr/test/FlickrEDCAsActivityTest.java
deleted file mode 100644
index f1257e3..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-flickr/src/test/java/org/apache/streams/gnip/flickr/test/FlickrEDCAsActivityTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gnip.flickr.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.apache.streams.gnip.powertrack.GnipActivityFixer;
-import org.apache.streams.pojo.json.Activity;
-import org.json.JSONObject;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests conversion of FlickrEDC inputs to Activity
- */
-@Ignore("ignore until test resources are available.")
-public class FlickrEDCAsActivityTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(FlickrEDCAsActivityTest.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- @Test
- public void Tests() throws Exception
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = FlickrEDCAsActivityTest.class.getResourceAsStream("/FlickrEDC.xml");
- if(is == null) LOGGER.debug("null");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
- XmlMapper xmlMapper = new XmlMapper();
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- ObjectMapper jsonMapper = new ObjectMapper();
-
- try {
- while (br.ready()) {
- String line = br.readLine();
-
- Object activityObject = xmlMapper.readValue(line, Object.class);
-
- String jsonString = jsonMapper.writeValueAsString(activityObject);
-
- JSONObject jsonObject = new JSONObject(jsonString);
-
- JSONObject fixedObject = GnipActivityFixer.fix(jsonObject);
-
- Activity activity = null;
- try {
- activity = jsonMapper.readValue(fixedObject.toString(), Activity.class);
- } catch( Exception e ) {
- LOGGER.error(jsonObject.toString());
- LOGGER.error(fixedObject.toString());
- e.printStackTrace();
- Assert.fail();
- }
- }
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-flickr/src/test/java/org/apache/streams/gnip/flickr/test/FlickrEDCSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-flickr/src/test/java/org/apache/streams/gnip/flickr/test/FlickrEDCSerDeTest.java b/streams-contrib/streams-provider-gnip/gnip-edc-flickr/src/test/java/org/apache/streams/gnip/flickr/test/FlickrEDCSerDeTest.java
deleted file mode 100644
index 3b3a847..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-flickr/src/test/java/org/apache/streams/gnip/flickr/test/FlickrEDCSerDeTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gnip.flickr.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests serialization of FlickrEDC inputs
- */
-@Ignore("ignore until test resources are available.")
-public class FlickrEDCSerDeTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(FlickrEDCSerDeTest.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- @Test
- public void Tests() throws Exception
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = FlickrEDCSerDeTest.class.getResourceAsStream("/FlickrEDC.xml");
- if(is == null) LOGGER.debug("null");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
- XmlMapper xmlMapper = new XmlMapper();
- xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- xmlMapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, Boolean.FALSE);
-
- ObjectMapper jsonMapper = new ObjectMapper();
-
- try {
- while (br.ready()) {
- String line = br.readLine();
-
- Object activityObject = xmlMapper.readValue(line, Object.class);
-
- String jsonObject = jsonMapper.writeValueAsString(activityObject);
- }
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/pom.xml b/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/pom.xml
deleted file mode 100644
index 6045dfb..0000000
--- a/streams-contrib/streams-provider-gnip/gnip-edc-googleplus/pom.xml
+++ /dev/null
@@ -1,102 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-provider-gnip</artifactId>
- <version>0.4-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>gnip-edc-googleplus</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Gnip EDC Google+ Provider</description>
-
- <dependencies>
-
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-xml</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>jackson-xml-databind</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>gnip-powertrack</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
-
- </build>
-</project>
[6/6] incubator-streams git commit: Merge remote-tracking branch
'eponvert/STREAMS-416'
Posted by sb...@apache.org.
Merge remote-tracking branch 'eponvert/STREAMS-416'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/5fd26a67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/5fd26a67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/5fd26a67
Branch: refs/heads/master
Commit: 5fd26a677dd2c3a3266f178101445c0148180d02
Parents: bb26799 0b512d8
Author: Steve Blackmon @steveblackmon <sb...@apache.org>
Authored: Tue Oct 18 19:36:53 2016 -0500
Committer: Steve Blackmon @steveblackmon <sb...@apache.org>
Committed: Tue Oct 18 19:36:53 2016 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 5 -
.../streams-persist-cassandra/pom.xml | 168 ----
.../configuration/CassandraConfiguration.java | 81 --
.../model/CassandraActivityStreamsEntry.java | 45 -
.../CassandraActivityStreamsRepository.java | 176 ----
.../repository/impl/CassandraKeyspace.java | 61 --
.../impl/CassandraSubscriptionRepository.java | 69 --
.../spring/streams-cassandra-context.xml | 25 -
.../CassandraActivityStreamsRepositoryTest.java | 99 ---
.../impl/CassandraActivitySubscriptionTest.java | 54 --
.../streams-provider-datasift/README.md | 18 -
.../streams-provider-datasift/pom.xml | 242 ------
.../streams/datasift/csdl/DatasiftCsdlUtil.java | 132 ---
.../DatasiftActivitySerializerProcessor.java | 97 ---
.../DatasiftTypeConverterProcessor.java | 167 ----
.../datasift/provider/DatasiftConverter.java | 38 -
.../provider/DatasiftManagedSourceSetup.java | 94 ---
.../datasift/provider/DatasiftPushProvider.java | 270 ------
.../provider/DatasiftStreamConfigurator.java | 52 --
.../provider/DatasiftStreamProvider.java | 242 ------
.../streams/datasift/provider/ErrorHandler.java | 47 --
.../streams/datasift/provider/Subscription.java | 60 --
.../serializer/DatasiftActivitySerializer.java | 65 --
.../serializer/DatasiftEventClassifier.java | 53 --
.../DatasiftInstagramActivitySerializer.java | 125 ---
.../DatasiftInteractionActivitySerializer.java | 247 ------
.../DatasiftTwitterActivitySerializer.java | 271 ------
.../datasift/util/StreamsDatasiftMapper.java | 89 --
.../org/apache/streams/datasift/Datasift.json | 462 -----------
.../streams/datasift/DatasiftConfiguration.json | 136 ----
.../datasift/DatasiftPushConfiguration.json | 20 -
.../datasift/DatasiftStreamConfiguration.json | 20 -
.../streams/datasift/DatasiftWebhookData.json | 35 -
.../datasift/facebook/DatasiftFacebook.json | 125 ---
.../datasift/instagram/DatasiftInstagram.json | 183 -----
.../interaction/DatasiftInteraction.json | 97 ---
.../datasift/twitter/DatasiftTwitter.json | 370 ---------
.../datasift/twitter/DatasiftTwitterMedia.json | 132 ---
.../datasift/twitter/DatasiftTwitterUser.json | 73 --
.../src/main/resources/datasift.conf | 25 -
.../src/site/markdown/index.md | 27 -
.../com/datasift/test/DatasiftSerDeTest.java | 75 --
.../provider/DatasiftStreamProviderTest.java | 144 ----
.../datasift/provider/ErrorHandlerTest.java | 40 -
.../datasift/provider/SubscriptionTest.java | 58 --
.../DatasiftActivitySerializerIT.java | 110 ---
.../serializer/DatasiftEventClassifierTest.java | 71 --
.../DatasiftInstagramActivitySerializerIT.java | 57 --
...DatasiftInteractionActivitySerializerIT.java | 62 --
.../DatasiftTwitterActivitySerializerIT.java | 57 --
.../gnip-edc-facebook/pom.xml | 152 ----
.../test/FacebookEDCAsActivityTest.java | 93 ---
.../facebook/test/FacebookEDCSerDeTest.java | 74 --
.../gnip-edc-flickr/pom.xml | 139 ----
.../flickr/test/FlickrEDCAsActivityTest.java | 92 ---
.../gnip/flickr/test/FlickrEDCSerDeTest.java | 76 --
.../gnip-edc-googleplus/pom.xml | 102 ---
.../com/gplus/api/GPlusActivitySerializer.java | 93 ---
.../com/gplus/api/GPlusEDCAsActivityTest.java | 95 ---
.../gnip-edc-instagram/pom.xml | 116 ---
.../jsonschema/com/instagram/Instagram.json | 208 -----
.../com/instagram/test/InstagramSerDeTest.java | 70 --
.../gnip-edc-reddit/pom.xml | 103 ---
.../reddit/api/RedditActivitySerializer.java | 107 ---
.../reddit/api/RedditEDCAsActivityJSONTest.java | 97 ---
.../gnip-edc-youtube/pom.xml | 139 ----
.../java/com/gnip/test/YouTubeEDCSerDeTest.java | 79 --
.../com/gnip/test/YoutubeEDCAsActivityTest.java | 87 --
.../gnip-powertrack/README.md | 8 -
.../gnip-powertrack/pom.xml | 207 -----
.../ActivityXMLActivitySerializer.java | 240 ------
.../gnip/powertrack/GnipActivityFixer.java | 151 ----
.../PowerTrackActivitySerializer.java | 121 ---
.../src/main/jsonschema/com/gnip/Gnip.json | 815 -------------------
.../src/main/xmlschema/com/gnip/binding.xjb | 33 -
.../src/main/xmlschema/com/gnip/entry.xsd | 398 ---------
.../gnip-powertrack/src/site/markdown/index.md | 15 -
.../test/PowerTrackDeserializationTest.java | 55 --
streams-contrib/streams-provider-gnip/pom.xml | 67 --
79 files changed, 9503 deletions(-)
----------------------------------------------------------------------
[4/6] incubator-streams git commit: Resolves STREAMS-416
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
deleted file mode 100644
index fbd5e52..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.DataSiftClient;
-import com.datasift.client.DataSiftConfig;
-import com.datasift.client.core.Stream;
-import com.datasift.client.stream.DeletedInteraction;
-import com.datasift.client.stream.Interaction;
-import com.datasift.client.stream.StreamEventListener;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Queues;
-import com.typesafe.config.Config;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.datasift.DatasiftConfiguration;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * Requires Java Version 1.7!
- * {@code DatasiftStreamProvider} is an implementation of the {@link org.apache.streams.core.StreamsProvider} interface. The provider
- * uses the Datasift java api to make connections. A single provider creates one connection per StreamHash in the configuration.
- */
-public class DatasiftStreamProvider implements StreamsProvider {
-
- private final static String STREAMS_ID = "DatasiftStreamProvider";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamProvider.class);
-
- private DatasiftConfiguration config;
- private ConcurrentLinkedQueue<Interaction> interactions = new ConcurrentLinkedQueue<Interaction>();
- private Map<String, DataSiftClient> clients;
- private StreamEventListener eventListener;
- private ObjectMapper mapper;
-
- /**
- * Constructor that searches for available configurations
- *
- * @param listener {@link com.datasift.client.stream.StreamEventListener} that handles deletion notices received from twitter.
- */
- public DatasiftStreamProvider(StreamEventListener listener) {
- this(listener, null);
- }
-
- // to set up a webhook we need to be able to return a reference to this queue
- public Queue<Interaction> getInteractions() {
- return interactions;
- }
-
- /**
- * @param listener {@link com.datasift.client.stream.StreamEventListener} that handles deletion notices received from twitter.
- * @param config Configuration to use
- */
- public DatasiftStreamProvider(StreamEventListener listener, DatasiftConfiguration config) {
- if (config == null) {
- Config datasiftConfig = StreamsConfigurator.config.getConfig("datasift");
- this.config = DatasiftStreamConfigurator.detectConfiguration(datasiftConfig);
- } else {
- this.config = config;
- }
- this.eventListener = listener;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
-
- Preconditions.checkNotNull(this.config);
- Preconditions.checkNotNull(this.config.getStreamHash());
- Preconditions.checkNotNull(this.config.getStreamHash().get(0));
- Preconditions.checkNotNull(this.config.getApiKey());
- Preconditions.checkNotNull(this.config.getUserName());
- Preconditions.checkNotNull(this.clients);
-
- for (String hash : this.config.getStreamHash()) {
- startStreamForHash(hash);
- }
-
- }
-
- /**
- * Creates a connection to datasift and starts collection of data from the resulting string.
- *
- * @param streamHash
- */
- public void startStreamForHash(String streamHash) {
- shutDownStream(streamHash);
- DataSiftClient client = getNewClient(this.config.getUserName(), this.config.getApiKey());
- client.liveStream().onStreamEvent(this.eventListener);
- client.liveStream().onError(new ErrorHandler(this, streamHash));
-
- client.liveStream().subscribe(new Subscription(Stream.fromString(streamHash), this.interactions));
- synchronized (this.clients) {
- this.clients.put(streamHash, client);
- }
- }
-
- /**
- * Exposed for testing purposes.
- *
- * @param userName
- * @param apiKey
- * @return
- */
- protected DataSiftClient getNewClient(String userName, String apiKey) {
- return new DataSiftClient(new DataSiftConfig(userName, apiKey));
- }
-
-
- /**
- * If a stream has been opened for the supplied stream hash, that stream will be shutdown.
- *
- * @param streamHash
- */
- public void shutDownStream(String streamHash) {
- synchronized (clients) {
- if (!this.clients.containsKey(streamHash))
- return;
- DataSiftClient client = this.clients.get(streamHash);
- LOGGER.debug("Shutting down stream for hash: {}", streamHash);
- client.shutdown();
- this.clients.remove(client);
- }
- }
-
- /**
- * Shuts down all open streams from datasift.
- */
- public void stop() {
- synchronized (clients) {
- for (DataSiftClient client : this.clients.values()) {
- client.shutdown();
- }
- }
- }
-
- // PRIME EXAMPLE OF WHY WE NEED NEW INTERFACES FOR PROVIDERS
- @Override
- //This is a hack. It is only like this because of how perpetual streams work at the moment. Read list server to debate/vote for new interfaces.
- public StreamsResultSet readCurrent() {
- Queue<StreamsDatum> datums = Queues.newConcurrentLinkedQueue();
- StreamsDatum datum = null;
- Interaction interaction;
- while (!this.interactions.isEmpty()) {
- interaction = this.interactions.poll();
- try {
- datum = new StreamsDatum(this.mapper.writeValueAsString(interaction.getData()), interaction.getData().get("interaction").get("id").textValue());
- } catch (JsonProcessingException jpe) {
- LOGGER.error("Exception while converting Interaction to String : {}", jpe);
- }
- if (datum != null) {
- while (!datums.offer(datum)) {
- Thread.yield();
- }
- }
-
- }
- return new StreamsResultSet(datums);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return this.clients != null && this.clients.size() > 0;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.interactions = new ConcurrentLinkedQueue<Interaction>();
- this.clients = Maps.newHashMap();
- this.mapper = StreamsDatasiftMapper.getInstance();
- }
-
- @Override
- public void cleanUp() {
- stop();
- }
-
- public DatasiftConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(DatasiftConfiguration config) {
- this.config = config;
- }
-
-
- /**
- * THIS CLASS NEEDS TO BE REPLACED/OVERRIDDEN BY ALL USERS. TWITTERS TERMS OF SERVICE SAYS THAT EVERYONE MUST
- * DELETE TWEETS FROM THEIR DATA STORE IF THEY RECEIVE A DELETE NOTICE.
- */
- public static class DeleteHandler extends StreamEventListener {
-
- public void onDelete(DeletedInteraction di) {
- //go off and delete the interaction if you have it stored. This is a strict requirement!
- LOGGER.info("DELETED:\n " + di);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java
deleted file mode 100644
index d5f19d5..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.stream.ErrorListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Listens for exceptions from the datasift streams and resets connections on errors.
- */
-public class ErrorHandler extends ErrorListener {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ErrorHandler.class);
-
- private String streamHash;
- private DatasiftStreamProvider provider;
-
- public ErrorHandler(DatasiftStreamProvider provider, String streamHash) {
- this.provider = provider;
- this.streamHash = streamHash;
- }
-
- @Override
- public void exceptionCaught(Throwable throwable) {
- LOGGER.error("DatasiftClient received Exception : {}", throwable);
- LOGGER.info("Attempting to restart client for stream hash : {}", this.streamHash);
- this.provider.startStreamForHash(this.streamHash);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java
deleted file mode 100644
index 058ba1d..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.core.Stream;
-import com.datasift.client.stream.DataSiftMessage;
-import com.datasift.client.stream.Interaction;
-import com.datasift.client.stream.StreamSubscription;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Queue;
-
-/**
- * Adds incomming {@link com.datasift.client.stream.Interaction} to the queue for the provider.
- */
-public class Subscription extends StreamSubscription {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Subscription.class);
- private Queue<Interaction> sharedQueue;
-
- public Subscription(Stream stream, Queue<Interaction> sharedQueue) {
- super(stream);
- this.sharedQueue = sharedQueue;
- }
-
- @Override
- public void onDataSiftLogMessage(DataSiftMessage dataSiftMessage) {
- if (dataSiftMessage.isError()) //should we restart the subscription on error?
- LOGGER.error("Datasift Error : {}", dataSiftMessage.getMessage());
- else if (dataSiftMessage.isWarning())
- LOGGER.warn("Datasift Warning : {}", dataSiftMessage.getMessage());
- else
- LOGGER.info("Datasift Info : {}", dataSiftMessage.getMessage());
- }
-
- @Override
- public void onMessage(Interaction interaction) {
- while (!this.sharedQueue.offer(interaction)) {
- Thread.yield();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
deleted file mode 100644
index b587cd6..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.json.Activity;
-
-import java.util.List;
-
-/**
- *
- */
-public class DatasiftActivitySerializer implements ActivitySerializer<Datasift> {
-
- private static final ObjectMapper MAPPER = StreamsDatasiftMapper.getInstance();
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public Datasift serialize(Activity deserialized) throws ActivitySerializerException {
- return null;
- }
-
- @Override
- public Activity deserialize(Datasift serialized) throws ActivitySerializerException {
- ActivitySerializer serializer = DatasiftEventClassifier.bestSerializer(serialized);
- return serializer.deserialize(serialized);
- }
-
- public Activity deserialize(String json) throws ActivitySerializerException {
- try {
- return deserialize(MAPPER.readValue(json, Datasift.class));
- } catch (Exception e) {
- throw new ActivitySerializerException(e);
- }
- }
-
- @Override
- public List<Activity> deserializeAll(List<Datasift> serializedList) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
deleted file mode 100644
index 7d7d547..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.instagram.Instagram;
-import org.apache.streams.datasift.interaction.Interaction;
-import org.apache.streams.datasift.twitter.Twitter;
-
-/**
- * Created by sblackmon on 11/6/14.
- */
-public class DatasiftEventClassifier {
-
- public static Class detectClass(Datasift event) {
-
- if(event.getTwitter() != null) {
- return Twitter.class;
- } else if(event.getInstagram() != null) {
- return Instagram.class;
- } else {
- return Interaction.class;
- }
- }
-
- public static ActivitySerializer bestSerializer(Datasift event) {
-
- if(event.getTwitter() != null) {
- return DatasiftTwitterActivitySerializer.getInstance();
- } else if(event.getInstagram() != null) {
- return DatasiftInstagramActivitySerializer.getInstance();
- } else {
- return DatasiftInteractionActivitySerializer.getInstance();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
deleted file mode 100644
index 1ed805d..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-package org.apache.streams.datasift.serializer;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.collect.Lists;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.instagram.From;
-import org.apache.streams.datasift.instagram.Instagram;
-import org.apache.streams.instagram.serializer.util.InstagramActivityUtil;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Image;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
-
-/**
- *
- */
-public class DatasiftInstagramActivitySerializer extends DatasiftInteractionActivitySerializer {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftInstagramActivitySerializer.class);
-
- private static DatasiftInstagramActivitySerializer instance = new DatasiftInstagramActivitySerializer();
-
- public static DatasiftInstagramActivitySerializer getInstance() {
- return instance;
- }
-
- @Override
- public Activity convert(Datasift event) {
- Activity activity = super.convert(event);
-
- Instagram instagram = event.getInstagram();
-
- activity.setActor(buildActor(event, instagram));
- activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
- activity.setProvider(InstagramActivityUtil.getProvider());
- activity.setLinks(getLinks(event.getInstagram()));
-
- activity.setVerb(selectVerb(event));
- addInstagramExtensions(activity, instagram);
- return activity;
- }
-
- /**
- * Gets links from the object
- * @return
- */
- private List<String> getLinks(Instagram instagram) {
- List<String> result = Lists.newLinkedList();
- if( instagram.getMedia() != null ) {
- result.add(instagram.getMedia().getImage());
- result.add(instagram.getMedia().getVideo());
- }
- return result;
- }
-
- public Actor buildActor(Datasift event, Instagram instagram) {
- Actor actor = super.buildActor(event.getInteraction());
- From user = instagram.getFrom();
-
- actor.setDisplayName(firstStringIfNotNull(user.getFullName()));
- actor.setId(formatId(Optional.fromNullable(
- user.getId())
- .or(actor.getId())));
-
- Image profileImage = new Image();
- String profileUrl = null;
- profileUrl = user.getProfilePicture();
- profileImage.setUrl(profileUrl);
- actor.setImage(profileImage);
-
- return actor;
- }
-
- public void addInstagramExtensions(Activity activity, Instagram instagram) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- List<String> hashTags;
- if(instagram.getMedia() != null) {
- hashTags = instagram.getMedia().getTags();
- extensions.put("hashtags", hashTags);
- extensions.put("keywords", activity.getContent());
- } else {
- extensions.put("keywords", activity.getContent());
-
- }
-
- }
-
- private String selectVerb(Datasift event) {
- if( event.getInteraction().getSubtype().equals("like"))
- return "like";
- else
- return "post";
- }
-
- public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:instagram", idparts));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java
deleted file mode 100644
index 79bf940..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.interaction.Interaction;
-import org.apache.streams.datasift.links.Links;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Generator;
-import org.apache.streams.pojo.json.Icon;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
-
-/**
- *
- */
-public class DatasiftInteractionActivitySerializer implements ActivitySerializer<Datasift>, Serializable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftInteractionActivitySerializer.class);
-
- private static DatasiftInteractionActivitySerializer instance = new DatasiftInteractionActivitySerializer();
-
- public static DatasiftInteractionActivitySerializer getInstance() {
- return instance;
- }
-
- ObjectMapper mapper = StreamsDatasiftMapper.getInstance();
-
- @Override
- public String serializationFormat() {
- return "application/json+datasift.com.v1.1";
- }
-
- @Override
- public Datasift serialize(Activity deserialized) {
- throw new UnsupportedOperationException("Cannot currently serialize to Datasift JSON");
- }
-
- public Activity deserialize(String datasiftJson) {
- try {
- return deserialize(this.mapper.readValue(datasiftJson, Datasift.class));
- } catch (Exception e) {
- LOGGER.error("Exception while trying convert,\n {},\n to a Datasift object.", datasiftJson);
- LOGGER.error("Exception : {}", e);
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Activity deserialize(Datasift serialized) {
-
- try {
-
- Activity activity = convert(serialized);
-
- return activity;
-
- } catch (Exception e) {
- throw new IllegalArgumentException("Unable to deserialize", e);
- }
-
- }
-
- @Override
- public List<Activity> deserializeAll(List<Datasift> datasifts) {
- List<Activity> activities = Lists.newArrayList();
- for( Datasift datasift : datasifts ) {
- activities.add(deserialize(datasift));
- }
- return activities;
- }
-
- public static Generator buildGenerator(Interaction interaction) {
- Generator generator = new Generator();
- generator.setDisplayName(interaction.getSource());
- generator.setId(interaction.getSource());
- return generator;
- }
-
- public static Icon getIcon(Interaction interaction) {
- return null;
- }
-
- public static Provider buildProvider(Interaction interaction) {
- Provider provider = new Provider();
- provider.setId("id:providers:"+interaction.getType());
- provider.setDisplayName(interaction.getType());
- return provider;
- }
-
- public static String getUrls(Interaction interaction) {
- return null;
- }
-
- public static void addDatasiftExtension(Activity activity, Datasift datasift) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- extensions.put("datasift", datasift);
- }
-
- public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:datasift", idparts));
- }
-
- public Activity convert(Datasift event) {
-
- Preconditions.checkNotNull(event);
- Preconditions.checkNotNull(event.getInteraction());
-
- Activity activity = new Activity();
- activity.setActor(buildActor(event.getInteraction()));
- activity.setVerb(selectVerb(event));
- activity.setObject(buildActivityObject(event.getInteraction()));
- activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
- activity.setTarget(buildTarget(event.getInteraction()));
- activity.setPublished(event.getInteraction().getCreatedAt());
- activity.setGenerator(buildGenerator(event.getInteraction()));
- activity.setIcon(getIcon(event.getInteraction()));
- activity.setProvider(buildProvider(event.getInteraction()));
- activity.setTitle(event.getInteraction().getTitle());
- activity.setContent(event.getInteraction().getContent());
- activity.setUrl(event.getInteraction().getLink());
- activity.setLinks(getLinks(event));
- addDatasiftExtension(activity, event);
- if( event.getInteraction().getGeo() != null) {
- addLocationExtension(activity, event.getInteraction());
- }
- return activity;
- }
-
- private String selectVerb(Datasift event) {
- return "post";
- }
-
- public Actor buildActor(Interaction interaction) {
- Actor actor = new Actor();
- org.apache.streams.datasift.interaction.Author author = interaction.getAuthor();
- if(author == null) {
- LOGGER.warn("Interaction does not contain author information.");
- return actor;
- }
- String userName = author.getUsername();
- String name = author.getName();
- Long id = author.getId();
- if(userName != null) {
- actor.setDisplayName(userName);
- } else {
- actor.setDisplayName(name);
- }
-
- if(id != null) {
- actor.setId(id.toString());
- } else {
- if(userName != null)
- actor.setId(userName);
- else
- actor.setId(name);
- }
- Image image = new Image();
- image.setUrl(interaction.getAuthor().getAvatar());
- actor.setImage(image);
- if (interaction.getAuthor().getLink()!=null){
- actor.setUrl(interaction.getAuthor().getLink());
- }
- return actor;
- }
-
- public static ActivityObject buildActivityObject(Interaction interaction) {
- ActivityObject actObj = new ActivityObject();
- actObj.setObjectType(interaction.getContenttype());
- actObj.setUrl(interaction.getLink());
- actObj.setId(formatId("post",interaction.getId()));
- actObj.setContent(interaction.getContent());
-
- return actObj;
- }
-
- public static List<String> getLinks(Datasift event) {
- List<String> result = Lists.newArrayList();
- Links links = event.getLinks();
- if(links == null)
- return null;
- for(Object link : links.getNormalizedUrl()) {
- if(link != null) {
- if(link instanceof String) {
- result.add((String) link);
- } else {
- LOGGER.warn("link is not of type String : {}", link.getClass().getName());
- }
- }
- }
- return result;
- }
-
- public static ActivityObject buildTarget(Interaction interaction) {
- return null;
- }
-
- public static void addLocationExtension(Activity activity, Interaction interaction) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- Map<String, Object> location = new HashMap<String, Object>();
- Map<String, Double> coordinates = new HashMap<String, Double>();
- coordinates.put("latitude", interaction.getGeo().getLatitude());
- coordinates.put("longitude", interaction.getGeo().getLongitude());
- location.put("coordinates", coordinates);
- extensions.put("location", location);
- }
-
- public static String firstStringIfNotNull(List<Object> list) {
- if( list != null && list.size() > 0) {
- return (String) list.get(0);
- } else return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java
deleted file mode 100644
index cbd29dd..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-package org.apache.streams.datasift.serializer;
-
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.interaction.Author;
-import org.apache.streams.datasift.interaction.Interaction;
-import org.apache.streams.datasift.twitter.DatasiftTwitterUser;
-import org.apache.streams.datasift.twitter.Retweet;
-import org.apache.streams.datasift.twitter.Twitter;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.twitter.converter.util.TwitterActivityUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class DatasiftTwitterActivitySerializer extends DatasiftInteractionActivitySerializer {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftTwitterActivitySerializer.class);
-
- private static DatasiftTwitterActivitySerializer instance = new DatasiftTwitterActivitySerializer();
-
- public static DatasiftTwitterActivitySerializer getInstance() {
- return instance;
- }
-
- @Override
- public Activity convert(Datasift event) {
- Activity activity = new Activity();
- Twitter twitter = event.getTwitter();
- boolean retweet = twitter.getRetweet() != null;
-
- activity.setActor(buildActor(event, twitter)); //TODO
- if(retweet) {
- activity.setVerb("share");
- } else {
- activity.setVerb("post");
- }
- activity.setObject(buildActivityObject(event.getInteraction()));
- activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
- activity.setTarget(buildTarget(event.getInteraction()));
- activity.setPublished(event.getInteraction().getCreatedAt());
- activity.setGenerator(buildGenerator(event.getInteraction()));
- activity.setIcon(getIcon(event.getInteraction()));
- activity.setProvider(TwitterActivityUtil.getProvider());
- activity.setTitle(event.getInteraction().getTitle());
- activity.setContent(event.getInteraction().getContent());
- activity.setUrl(event.getInteraction().getLink());
- if(retweet)
- activity.setLinks(getLinks(twitter.getRetweet()));
- else
- activity.setLinks(getLinks(twitter));
- addDatasiftExtension(activity, event);
- if( twitter.getGeo() != null) {
- addLocationExtension(activity, twitter);
- }
- addTwitterExtensions(activity, twitter, event.getInteraction());
- return activity;
- }
-
- /**
- * Get the links from this tweet as a list
- * @param twitter
- * @return the links from the tweet
- */
- public List<String> getLinks(Twitter twitter) {
- return getLinks(twitter.getLinks());
- }
-
- /**
- * Get the links from this tweet as a list
- * @param retweet
- * @return the links from the tweet
- */
- public List<String> getLinks(Retweet retweet) {
- return getLinks(retweet.getLinks());
- }
-
- /**
- * Converts the list of objects to a list of strings
- * @param links
- * @return
- */
- private List<String> getLinks(List<Object> links) {
- if(links == null)
- return Lists.newArrayList();
- List<String> result = Lists.newLinkedList();
- for(Object obj : links) {
- if(obj instanceof String) {
- result.add((String) obj);
- } else {
- LOGGER.warn("Links is not instance of String : {}", obj.getClass().getName());
- }
- }
- return result;
- }
-
- public Actor buildActor(Datasift event, Twitter twitter) {
- DatasiftTwitterUser user = twitter.getUser();
- Actor actor = super.buildActor(event.getInteraction());
- if(user == null) {
- user = twitter.getRetweet().getUser();
- }
-
- actor.setDisplayName(user.getName());
- actor.setId(formatId(Optional.fromNullable(
- user.getIdStr())
- .or(Optional.of(user.getId().toString()))
- .orNull()));
- actor.setSummary(user.getDescription());
- try {
- actor.setPublished(user.getCreatedAt());
- } catch (Exception e) {
- LOGGER.warn("Exception trying to parse date : {}", e);
- }
-
- if(user.getUrl() != null) {
- actor.setUrl(user.getUrl());
- }
-
- Map<String, Object> extensions = new HashMap<String,Object>();
- extensions.put("location", user.getLocation());
- extensions.put("posts", user.getStatusesCount());
- extensions.put("followers", user.getFollowersCount());
- extensions.put("screenName", user.getScreenName());
- if(user.getAdditionalProperties() != null) {
- extensions.put("favorites", user.getFavouritesCount());
- }
-
- Image profileImage = new Image();
- String profileUrl = null;
- Author author = event.getInteraction().getAuthor();
- if( author != null )
- profileUrl = author.getAvatar();
- if(profileUrl == null && user.getProfileImageUrlHttps() != null) {
- Object url = user.getProfileImageUrlHttps();
- if(url instanceof String)
- profileUrl = (String) url;
- }
- if(profileUrl == null) {
- profileUrl = user.getProfileImageUrl();
- }
- profileImage.setUrl(profileUrl);
- actor.setImage(profileImage);
-
- actor.setAdditionalProperty("extensions", extensions);
- return actor;
- }
-
- public void addLocationExtension(Activity activity, Twitter twitter) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- Map<String, Object> location = Maps.newHashMap();
- double[] coordiantes = new double[] { twitter.getGeo().getLongitude(), twitter.getGeo().getLatitude() };
- Map<String, Object> coords = Maps.newHashMap();
- coords.put("coordinates", coordiantes);
- coords.put("type", "geo_point");
- location.put("coordinates", coords);
- extensions.put("location", location);
- }
-
- public void addTwitterExtensions(Activity activity, Twitter twitter, Interaction interaction) {
- Retweet retweet = twitter.getRetweet();
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- List<String> hashTags = Lists.newLinkedList();
- List<Object> hts = Lists.newLinkedList();
- if(twitter.getHashtags() != null) {
- hts = twitter.getHashtags();
- } else if (retweet != null) {
- hts = retweet.getHashtags();
- }
- if(hts != null) {
- for(Object ht : twitter.getHashtags()) {
- if(ht instanceof String) {
- hashTags.add((String) ht);
- } else {
- LOGGER.warn("Hashtag was not instance of String : {}", ht.getClass().getName());
- }
- }
- }
- extensions.put("hashtags", hashTags);
-
-
- if(retweet != null) {
- Map<String, Object> rebroadcasts = Maps.newHashMap();
- rebroadcasts.put("perspectival", true);
- rebroadcasts.put("count", retweet.getCount());
- extensions.put("rebroadcasts", rebroadcasts);
- }
-
- if(interaction.getAdditionalProperties() != null) {
- ArrayList<Map<String,Object>> userMentions = createUserMentions(interaction);
-
- if(userMentions.size() > 0)
- extensions.put("user_mentions", userMentions);
- }
-
- extensions.put("keywords", interaction.getContent());
- }
-
- /**
- * Returns an ArrayList of all UserMentions in this interaction
- * Note: The ID list and the handle lists do not necessarily correspond 1:1 for this provider
- * If those lists are the same size, then they will be merged into individual UserMention
- * objects. However, if they are not the same size, a new UserMention object will be created
- * for each entry in both lists.
- *
- * @param interaction
- * @return
- */
- private ArrayList<Map<String,Object>> createUserMentions(Interaction interaction) {
- ArrayList<String> mentions = (ArrayList<String>) interaction.getAdditionalProperties().get("mentions");
- ArrayList<Long> mentionIds = (ArrayList<Long>) interaction.getAdditionalProperties().get("mention_ids");
- ArrayList<Map<String,Object>> userMentions = new ArrayList<Map<String,Object>>();
-
- if(mentions != null && !mentions.isEmpty()) {
- for(int x = 0; x < mentions.size(); x ++) {
- Map<String, Object> actor = new HashMap<String, Object>();
- actor.put("displayName", mentions.get(x));
- actor.put("handle", mentions.get(x));
-
- userMentions.add(actor);
- }
- }
- if(mentionIds != null && !mentionIds.isEmpty()) {
- for(int x = 0; x < mentionIds.size(); x ++) {
- Map<String, Object> actor = new HashMap<String, Object>();
- actor.put("id", "id:twitter:" + mentionIds.get(x));
-
- userMentions.add(actor);
- }
- }
-
- return userMentions;
- }
-
- public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java
deleted file mode 100644
index 93ab28b..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.util;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import org.apache.streams.data.util.RFC3339Utils;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
-import java.io.IOException;
-
-/**
- * Created by sblackmon on 3/27/14.
- *
- * Depracated: Use StreamsJacksonMapper instead
- */
-@Deprecated()
-public class StreamsDatasiftMapper extends StreamsJacksonMapper {
-
- public static final String DATASIFT_FORMAT = "EEE, dd MMM yyyy HH:mm:ss Z";
-
- public static final DateTimeFormatter DATASIFT_FORMATTER = DateTimeFormat.forPattern(DATASIFT_FORMAT);
-
- public static final Long getMillis(String dateTime) {
-
- // this function is for pig which doesn't handle exceptions well
- try {
- Long result = DATASIFT_FORMATTER.parseMillis(dateTime);
- return result;
- } catch( Exception e ) {
- return null;
- }
-
- }
-
- private static final StreamsDatasiftMapper INSTANCE = new StreamsDatasiftMapper();
-
- public static StreamsDatasiftMapper getInstance(){
- return INSTANCE;
- }
-
- public StreamsDatasiftMapper() {
- super();
- registerModule(new SimpleModule()
- {
- {
- addDeserializer(DateTime.class, new StdDeserializer<DateTime>(DateTime.class) {
- @Override
- public DateTime deserialize(JsonParser jpar, DeserializationContext context) throws IOException, JsonProcessingException {
- DateTime result = null;
- try {
- result = DATASIFT_FORMATTER.parseDateTime(jpar.getValueAsString());
- } catch (Exception e) {}
- if (result == null) {
- try {
- result = RFC3339Utils.getInstance().parseToUTC(jpar.getValueAsString());
- } catch (Exception e) {}
- }
- return result;
- }
- });
- }
- });
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json
deleted file mode 100644
index 52d9912..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json
+++ /dev/null
@@ -1,462 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.datasift.Datasift",
- "properties": {
- "blog": {
- "javaType": "org.apache.streams.datasift.blog.Blog",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "type": "object",
- "javaType": "org.apache.streams.datasift.blog.BlogAuthor",
- "dynamic": "true",
- "properties": {
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "domain": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- },
- "demographic": {
- "dynamic": "true",
- "properties": {
- "gender": {
- "type": "string"
- }
- }
- },
- "facebook": {
- "type": "object",
- "$ref": "facebook/DatasiftFacebook.json"
- },
- "instagram": {
- "type": "object",
- "$ref": "instagram/DatasiftInstagram.json"
- },
- "interaction": {
- "type": "object",
- "$ref": "interaction/DatasiftInteraction.json"
- },
- "klout": {
- "javaType": "org.apache.streams.datasift.interaction.Klout",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "score": {
- "type": "integer"
- }
- }
- },
- "language": {
- "javaType": "org.apache.streams.datasift.language.Language",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "confidence": {
- "type": "integer"
- },
- "tag": {
- "type": "string"
- },
- "tag_extended": {
- "type": "string"
- }
- }
- },
- "links": {
- "javaType": "org.apache.streams.datasift.links.Links",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "code": {
- "type": "array",
- "items": [
- {
- "type": "integer"
- }
- ]
- },
- "created_at": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "hops": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "meta": {
- "dynamic": "true",
- "javaType": "org.apache.streams.datasift.links.Meta",
- "type": "object",
- "properties": {
- "charset": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "content_type": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "description": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "keywords": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "lang": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "opengraph": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.interaction.meta.OpenGraph",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "description": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "site_name": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- },
- "twitter": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.interaction.meta.Twitter",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "card": {
- "type": "string"
- },
- "creator": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "player": {
- "type": "string"
- },
- "player_height": {
- "type": "string"
- },
- "player_width": {
- "type": "string"
- },
- "site": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- }
- }
- },
- "normalized_url": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "retweet_count": {
- "type": "array",
- "items": [
- {
- "type": "long"
- }
- ]
- },
- "title": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "url": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "salience": {
- "javaType": "org.apache.streams.datasift.salience.Salience",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "content": {
- "javaType": "org.apache.streams.datasift.salience.content.Content",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "entities": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.salience.content.Entities",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "about": {
- "type": "integer"
- },
- "confident": {
- "type": "integer"
- },
- "evidence": {
- "type": "integer"
- },
- "label": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "sentiment": {
- "type": "integer"
- },
- "themes": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- }
- ]
- },
- "sentiment": {
- "type": "integer"
- }
- }
- },
- "title": {
- "javaType": "org.apache.streams.datasift.salience.title.Title",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "entities": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.salience.title.Entities",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "about": {
- "type": "integer"
- },
- "confident": {
- "type": "integer"
- },
- "evidence": {
- "type": "integer"
- },
- "label": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "sentiment": {
- "type": "integer"
- },
- "themes": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- }
- ]
- },
- "sentiment": {
- "type": "integer"
- }
- }
- }
- }
- },
- "trends": {
- "javaType": "org.apache.streams.datasift.trends.Trends",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "content": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "source": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "type": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "twitter": {
- "type": "object",
- "$ref": "twitter/DatasiftTwitter.json"
- },
- "youtube": {
- "javaType": "org.apache.streams.datasift.youtube.YouTube",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "javaType": "org.apache.streams.datasift.youtube.Author",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "commentslink": {
- "type": "string"
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "videolink": {
- "type": "string"
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json
deleted file mode 100644
index 99f889e..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json
+++ /dev/null
@@ -1,136 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.datasift.DatasiftConfiguration",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "apiKey": {
- "type": "string"
- },
- "userName": {
- "type": "string"
- },
- "streamHash": {
- "type": "array",
- "minItems": 0,
- "items": {
- "type": "string"
- }
- },
- "streamCsdl": {
- "type": "array",
- "minItems": 0,
- "items": {
- "type": "string"
- }
- },
- "managedSources": {
- "type": "array",
- "minItems": 0,
- "items": {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSource",
- "type": "object",
- "properties": {
- "auth": {
- "type": "array",
- "items": {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceAuth",
- "type": "object",
- "properties": {
- "identity_id": {
- "type": "string"
- },
- "parameters": {
- "type": "object",
- "properties": {
- "value": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "created_at": {
- "type": "number"
- },
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "parameters": {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceParams",
- "type": "object",
- "properties": {
- "comments": {
- "type": "boolean"
- },
- "likes": {
- "type": "boolean"
- }
- }
- },
- "resources": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceResource",
- "type": "object",
- "properties": {
- "parameters": {
- "type": "object",
- "properties": {
- "distance": {
- "type": "number"
- },
- "exact_match": {
- "type": "string"
- },
- "foursq": {
- "type": "string"
- },
- "lat": {
- "type": "number"
- },
- "lng": {
- "type": "number"
- },
- "type": {
- "type": "string",
- "enum" : ["user", "tag", "area", "location", "popular"]
- },
- "usermeta": {
- "type": "string"
- },
- "value": {
- "type": "number"
- }
- }
- },
- "resource_id": {
- "type": "string"
- }
- }
- }
- ]
- },
- "source_type": {
- "type": "string",
- "enum" : ["facebook_page", "googleplus", "instagram", "yammer"]
- },
- "status": {
- "type": "string"
- }
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json
deleted file mode 100644
index 4c9fe5c..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.DatasiftPushConfiguration",
- "extends": {"$ref":"DatasiftConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "streamHash": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json
deleted file mode 100644
index 5c43fea..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.DatasiftStreamConfiguration",
- "extends": {"$ref":"DatasiftConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "streamHash": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json
deleted file mode 100644
index 7f3bee6..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "javaType": "org.apache.streams.datasift.DatasiftWebhookData",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "string"
- },
- "hash": {
- "type": "string"
- },
- "hash_type": {
- "type": "string"
- },
- "count": {
- "type": "long"
- },
- "delivered_at": {
- "type": "string",
- "format": "date-time"
- },
- "interactions": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.datasift.Datasift",
- "javaInterfaces": ["java.io.Serializable"]
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json
deleted file mode 100644
index 6085124..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json
+++ /dev/null
@@ -1,125 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.facebook.Facebook",
- "javaInterfaces": ["java.io.Serializable"],
- "properties" :
- {
- "application" :
- {
- "type" : "string"
- },
- "author" :
- {
- "javaType" : "org.apache.streams.datasift.facebook.Author",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "avatar" :
- {
- "type" : "string"
- },
- "id" :
- {
- "type" : "string"
- },
- "link" :
- {
- "type" : "string"
- },
- "name" :
- {
- "type" : "string"
- }
- }
- },
- "created_at" :
- {
- "type" : "string",
- "format" : "date-time"
- },
- "id" :
- {
- "type" : "string"
- },
- "likes" :
- {
- "javaType" : "org.apache.streams.datasift.facebook.Likes",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "count" :
- {
- "type" : "integer"
- },
- "ids" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "names" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- }
- }
- },
- "message" :
- {
- "type" : "string"
- },
- "source" :
- {
- "type" : "string"
- },
- "to" :
- {
- "javaType" : "org.apache.streams.datasift.facebook.To",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "ids" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "names" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- }
- }
- },
- "type" :
- {
- "type" : "string"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json
deleted file mode 100644
index 8c209b1..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json
+++ /dev/null
@@ -1,183 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.instagram.Instagram",
- "javaInterfaces": ["java.io.Serializable"],
- "properties" :
- {
- "caption" :
- {
- "type" : "object",
- "properties" :
- {
- "created_time" :
- {
- "type" : "number"
- },
- "id" :
- {
- "type" : "string"
- },
- "text" :
- {
- "type" : "string"
- },
- "from" :
- {
- "type" : "object",
- "properties" :
- {
- "full_name" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "id" :
- {
- "type" : "string"
- },
- "profile_picture" :
- {
- "type" : "string"
- },
- "username" :
- {
- "type" : "string"
- }
- }
- }
- }
- },
- "created_at" :
- {
- "type" : "string",
- "format" : "date-time"
- },
- "from" :
- {
- "javaType" : "org.apache.streams.datasift.instagram.From",
- "type" : "object",
- "properties" :
- {
- "full_name" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "id" :
- {
- "type" : "string"
- },
- "profile_picture" :
- {
- "type" : "string"
- },
- "username" :
- {
- "type" : "string"
- }
- }
- },
- "id" :
- {
- "type" : "string"
- },
- "images" :
- {
- "type" : "object",
- "properties" :
- {
- "low_resolution" :
- {
- "type" : "object"
- },
- "standard_resolution" :
- {
- "type" : "object"
- },
- "thumbnail" :
- {
- "type" : "object"
- }
- }
- },
- "media" :
- {
- "javaType" : "org.apache.streams.datasift.instagram.Media",
- "type" : "object",
- "properties" :
- {
- "caption" :
- {
- "type" : "string"
- },
- "created_time" :
- {
- "type" : "number"
- },
- "filter" :
- {
- "type" : "string"
- },
- "id" :
- {
- "type" : "string"
- },
- "image" :
- {
- "type" : "string"
- },
- "link" :
- {
- "type" : "string"
- },
- "tags" :
- {
- "type" : "array",
- "items" :
- {
- "type" : "string"
- }
- },
- "type" :
- {
- "type" : "string"
- },
- "username" :
- {
- "type" : "string"
- },
- "video" :
- {
- "type" : "string"
- }
- }
- },
- "tags" :
- {
- "type" : "array",
- "items" :
- {
- "type" : "string"
- }
- },
- "type" :
- {
- "type" : "string"
- }
- }
-}
\ No newline at end of file