Added: incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java
URL: http://svn.apache.org/viewvc/incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java?rev=1556893&view=auto
==============================================================================
--- incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java (added)
+++ incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java Thu Jan 9 17:27:50 2014
@@ -0,0 +1,90 @@
+package org.apache.streams.kafka;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.typesafe.config.Config;
+import kafka.javaapi.producer.Producer;
+import kafka.producer.KeyedMessage;
+import kafka.producer.ProducerConfig;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsPersistWriter;
+import org.apache.streams.util.GuidUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.Properties;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+public class KafkaPersistWriter implements StreamsPersistWriter, Serializable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(KafkaPersistWriter.class);
+
+ protected volatile Queue<StreamsDatum> persistQueue;
+
+ private ObjectMapper mapper = new ObjectMapper();
+
+ private KafkaConfiguration config;
+
+ private Producer<String, StreamsDatum> producer;
+
+ public KafkaPersistWriter() {
+ Config config = StreamsConfigurator.config.getConfig("kafka");
+ this.config = KafkaConfigurator.detectConfiguration(config);
+ this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+ }
+
+ public KafkaPersistWriter(Queue<StreamsDatum> persistQueue) {
+ Config config = StreamsConfigurator.config.getConfig("kafka");
+ this.config = KafkaConfigurator.detectConfiguration(config);
+ this.persistQueue = persistQueue;
+ }
+
+ public KafkaPersistWriter(KafkaConfiguration config) {
+ this.config = config;
+ this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+ }
+
+ public KafkaPersistWriter(KafkaConfiguration config, Queue<StreamsDatum> persistQueue) {
+ this.config = config;
+ this.persistQueue = persistQueue;
+ }
+
+ @Override
+ public void start() {
+ Properties props = new Properties();
+
+ props.put("metadata.broker.list", config.getBrokerlist());
+ props.put("serializer.class", "kafka.serializer.StringEncoder");
+ props.put("partitioner.class", "org.apache.streams.kafka.StreamsPartitioner");
+ props.put("request.required.acks", "1");
+
+ ProducerConfig config = new ProducerConfig(props);
+
+ producer = new Producer<String, StreamsDatum>(config);
+ }
+
+ @Override
+ public void stop() {
+ producer.close();
+ }
+
+ @Override
+ public void write(StreamsDatum entry) {
+
+ try {
+ String text = mapper.writeValueAsString(entry);
+
+ String hash = GuidUtils.generateGuid(text);
+
+ KeyedMessage<String, StreamsDatum> data = new KeyedMessage<String, StreamsDatum>(config.getTopic(), hash, entry);
+
+ producer.send(data);
+
+ } catch (JsonProcessingException e) {
+ LOGGER.warn("save: {}", e);
+ }// put
+ }
+}
Added: incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriterTask.java
URL: http://svn.apache.org/viewvc/incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriterTask.java?rev=1556893&view=auto
==============================================================================
--- incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriterTask.java (added)
+++ incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriterTask.java Thu Jan 9 17:27:50 2014
@@ -0,0 +1,34 @@
+package org.apache.streams.kafka;
+
+import org.apache.streams.core.StreamsDatum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Random;
+
+public class KafkaPersistWriterTask implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(KafkaPersistWriterTask.class);
+
+ private KafkaPersistWriter writer;
+
+ public KafkaPersistWriterTask(KafkaPersistWriter writer) {
+ this.writer = writer;
+ }
+
+ @Override
+ public void run() {
+
+ while(true) {
+ try {
+ StreamsDatum entry = writer.persistQueue.remove();
+ writer.write(entry);
+ Thread.sleep(new Random().nextInt(100));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ }
+
+}
Added: incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/StreamsPartitioner.java
URL: http://svn.apache.org/viewvc/incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/StreamsPartitioner.java?rev=1556893&view=auto
==============================================================================
--- incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/StreamsPartitioner.java (added)
+++ incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/StreamsPartitioner.java Thu Jan 9 17:27:50 2014
@@ -0,0 +1,23 @@
+package org.apache.streams.kafka;
+
+/**
+ * Created by sblackmon on 12/15/13.
+ */
+import kafka.producer.Partitioner;
+import kafka.utils.VerifiableProperties;
+
+public class StreamsPartitioner implements Partitioner<String> {
+ public StreamsPartitioner (VerifiableProperties props) {
+
+ }
+
+ public int partition(String key, int a_numPartitions) {
+ int partition = 0;
+ int offset = key.lastIndexOf('.');
+ if (offset > 0) {
+ partition = Integer.parseInt( key.substring(offset+1)) % a_numPartitions;
+ }
+ return partition;
+ }
+
+}
Added: incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/jsonschema/org/apache/streams/kafka/KafkaConfiguration.json
URL: http://svn.apache.org/viewvc/incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/jsonschema/org/apache/streams/kafka/KafkaConfiguration.json?rev=1556893&view=auto
==============================================================================
--- incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/jsonschema/org/apache/streams/kafka/KafkaConfiguration.json (added)
+++ incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/jsonschema/org/apache/streams/kafka/KafkaConfiguration.json Thu Jan 9 17:27:50 2014
@@ -0,0 +1,17 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.kafka.KafkaConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "brokerlist": {
+ "type": "string",
+ "description": "A comma-delimited list of broker nodes"
+ },
+ "topic": {
+ "type": "string",
+ "description": "A topic to write to by default"
+ }
+ }
+}
\ No newline at end of file
Added: incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/resources/reference.properties
URL: http://svn.apache.org/viewvc/incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/resources/reference.properties?rev=1556893&view=auto
==============================================================================
--- incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/resources/reference.properties (added)
+++ incubator/streams/branches/sblackmon/streams-contrib/streams-persist-kafka/src/main/resources/reference.properties Thu Jan 9 17:27:50 2014
@@ -0,0 +1,37 @@
+############################# Producer Basics #############################
+
+# list of brokers used for bootstrapping knowledge about the rest of the cluster
+# format: host1:port1,host2:port2 ...
+kafka.metadata.broker.list=localhost:9092
+
+# name of the partitioner class for partitioning events; default partition spreads data randomly
+#partitioner.class=
+
+# specifies whether the messages are sent asynchronously (async) or synchronously (sync)
+kafka.producer.type=sync
+
+# specify the compression codec for all data generated: none , gzip, snappy.
+# the old config values work as well: 0, 1, 2 for none, gzip, snappy, respectivally
+kafka.compression.codec=none
+
+# message encoder
+kafka.serializer.class=kafka.serializer.DefaultEncoder
+
+# allow topic level compression
+#compressed.topics=
+
+############################# Async Producer #############################
+# maximum time, in milliseconds, for buffering data on the producer queue
+#queue.buffering.max.ms=
+
+# the maximum size of the blocking queue for buffering on the producer
+#queue.buffering.max.messages=
+
+# Timeout for event enqueue:
+# 0: events will be enqueued immediately or dropped if the queue is full
+# -ve: enqueue will block indefinitely if the queue is full
+# +ve: enqueue will block up to this many milliseconds if the queue is full
+#queue.enqueue.timeout.ms=
+
+# the number of messages batched at the producer
+#batch.num.messages=
\ No newline at end of file
Added: incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/pom.xml
URL: http://svn.apache.org/viewvc/incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/pom.xml?rev=1556893&view=auto
==============================================================================
--- incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/pom.xml (added)
+++ incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/pom.xml Thu Jan 9 17:27:50 2014
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-contrib</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streams-provider-datasift</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema/com/datasift</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>com.datasift</targetPackage>
+ <useLongIntegers>true</useLongIntegers>
+ <useJodaDates>false</useJodaDates>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
Added: incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
URL: http://svn.apache.org/viewvc/incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json?rev=1556893&view=auto
==============================================================================
--- incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json (added)
+++ incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json Thu Jan 9 17:27:50 2014
@@ -0,0 +1,1205 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "javaType": "com.datasift.Datasift",
+ "properties": {
+ "blog": {
+ "javaType": "com.datasift.blog.Blog",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "author": {
+ "type": "object",
+ "javaType": "com.datasift.blog.BlogAuthor",
+ "dynamic": "true",
+ "properties": {
+ "link": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ }
+ }
+ },
+ "content": {
+ "type": "string"
+ },
+ "contenttype": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "domain": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "link": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ },
+ "demographic": {
+ "dynamic": "true",
+ "properties": {
+ "gender": {
+ "type": "string"
+ }
+ }
+ },
+ "facebook": {
+ "javaType": "com.datasift.config.Facebook",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "application": {
+ "type": "string"
+ },
+ "author": {
+ "javaType": "com.datasift.config.Author",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "avatar": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "link": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ }
+ }
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "likes": {
+ "javaType": "com.datasift.config.Likes",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "count": {
+ "type": "long"
+ },
+ "ids": {
+ "type": "string"
+ },
+ "names": {
+ "type": "string"
+ }
+ }
+ },
+ "message": {
+ "type": "string"
+ },
+ "source": {
+ "type": "string"
+ },
+ "to": {
+ "javaType": "com.datasift.config.To",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "ids": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "names": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ }
+ }
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ },
+ "interaction": {
+ "javaType": "com.datasift.interaction.Interaction",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "author": {
+ "javaType": "com.datasift.interaction.Author",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "avatar": {
+ "type": "string"
+ },
+ "id": {
+ "type": "long"
+ },
+ "link": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "content": {
+ "type": "string"
+ },
+ "contenttype": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "geo": {
+ "javaType": "com.datasift.interaction.Geo",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "latitude": {
+ "type": "double"
+ },
+ "longitude": {
+ "type": "double"
+ }
+ }
+ },
+ "id": {
+ "type": "string"
+ },
+ "link": {
+ "type": "string"
+ },
+ "schema": {
+ "dynamic": "true",
+ "properties": {
+ "version": {
+ "type": "long"
+ }
+ }
+ },
+ "source": {
+ "type": "string"
+ },
+ "tags": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "title": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ },
+ "klout": {
+ "javaType": "com.datasift.klout.Klout",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "score": {
+ "type": "long"
+ }
+ }
+ },
+ "language": {
+ "javaType": "com.datasift.interaction.Language",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "confidence": {
+ "type": "long"
+ },
+ "tag": {
+ "type": "string"
+ }
+ }
+ },
+ "links": {
+ "javaType": "com.datasift.interaction.Links",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "code": {
+ "type": "long"
+ },
+ "created_at": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "hops": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "meta": {
+ "dynamic": "true",
+ "javaType": "com.datasift.interaction.Meta",
+ "type": "object",
+ "properties": {
+ "charset": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "content_type": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "description": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "keywords": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "lang": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "opengraph": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "com.datasift.interaction.meta.OpenGraph",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "description": {
+ "type": "string"
+ },
+ "image": {
+ "type": "string"
+ },
+ "site_name": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "twitter": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "com.datasift.interaction.meta.Twitter",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "card": {
+ "type": "string"
+ },
+ "creator": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "image": {
+ "type": "string"
+ },
+ "player": {
+ "type": "string"
+ },
+ "player_height": {
+ "type": "string"
+ },
+ "player_width": {
+ "type": "string"
+ },
+ "site": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ "normalized_url": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "retweet_count": {
+ "type": "long"
+ },
+ "title": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "url": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ }
+ }
+ },
+ "salience": {
+ "javaType": "com.datasift.salience.Salience",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "content": {
+ "javaType": "com.datasift.salience.content.Content",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "entities": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "com.datasift.salience.content.Entities",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "about": {
+ "type": "long"
+ },
+ "confident": {
+ "type": "long"
+ },
+ "evidence": {
+ "type": "long"
+ },
+ "label": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "sentiment": {
+ "type": "long"
+ },
+ "themes": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "sentiment": {
+ "type": "long"
+ }
+ }
+ },
+ "title": {
+ "javaType": "com.datasift.salience.title.Title",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "entities": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "com.datasift.salience.title.Entities",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "about": {
+ "type": "long"
+ },
+ "confident": {
+ "type": "long"
+ },
+ "evidence": {
+ "type": "long"
+ },
+ "label": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "sentiment": {
+ "type": "long"
+ },
+ "themes": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "sentiment": {
+ "type": "long"
+ }
+ }
+ }
+ }
+ },
+ "trends": {
+ "javaType": "com.datasift.trends.Trends",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "content": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "source": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "type": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ }
+ }
+ },
+ "twitter": {
+ "javaType": "com.datasift.twitter.Twitter",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "domains": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "filter_level": {
+ "type": "string"
+ },
+ "geo": {
+ "javaType": "com.datasift.twitter.Geo",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "latitude": {
+ "type": "double"
+ },
+ "longitude": {
+ "type": "double"
+ }
+ }
+ },
+ "hashtags": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "id": {
+ "type": "string"
+ },
+ "in_reply_to_screen_name": {
+ "type": "string"
+ },
+ "in_reply_to_status_id": {
+ "type": "string"
+ },
+ "in_reply_to_user_id": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "links": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "media": {
+ "type": "array",
+ "items": [
+ {
+ "javaType": "com.datasift.twitter.Media",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "display_url": {
+ "type": "string"
+ },
+ "expanded_url": {
+ "type": "string"
+ },
+ "id": {
+ "type": "long"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "media_url": {
+ "type": "string"
+ },
+ "media_url_https": {
+ "type": "string"
+ },
+ "sizes": {
+ "dynamic": "true",
+ "properties": {
+ "large": {
+ "dynamic": "true",
+ "properties": {
+ "h": {
+ "type": "long"
+ },
+ "resize": {
+ "type": "string"
+ },
+ "w": {
+ "type": "long"
+ }
+ }
+ },
+ "medium": {
+ "dynamic": "true",
+ "properties": {
+ "h": {
+ "type": "long"
+ },
+ "resize": {
+ "type": "string"
+ },
+ "w": {
+ "type": "long"
+ }
+ }
+ },
+ "small": {
+ "dynamic": "true",
+ "properties": {
+ "h": {
+ "type": "long"
+ },
+ "resize": {
+ "type": "string"
+ },
+ "w": {
+ "type": "long"
+ }
+ }
+ },
+ "thumb": {
+ "dynamic": "true",
+ "properties": {
+ "h": {
+ "type": "long"
+ },
+ "resize": {
+ "type": "string"
+ },
+ "w": {
+ "type": "long"
+ }
+ }
+ }
+ }
+ },
+ "source_status_id": {
+ "type": "long"
+ },
+ "source_status_id_str": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ },
+ "mention_ids": {
+ "type": "long"
+ },
+ "mentions": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "place": {
+ "javaType": "com.datasift.twitter.Place",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "attributes": {
+ "dynamic": "true",
+ "properties": {
+ "street_address": {
+ "type": "string"
+ }
+ }
+ },
+ "country": {
+ "type": "string"
+ },
+ "country_code": {
+ "type": "string"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "place_type": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ },
+ "retweet": {
+ "javaType": "com.datasift.twitter.Retweet",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "count": {
+ "type": "long"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "domains": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "hashtags": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "id": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "links": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "media": {
+ "dynamic": "true",
+ "properties": {
+ "display_url": {
+ "type": "string"
+ },
+ "expanded_url": {
+ "type": "string"
+ },
+ "id": {
+ "type": "long"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "media_url": {
+ "type": "string"
+ },
+ "media_url_https": {
+ "type": "string"
+ },
+ "sizes": {
+ "dynamic": "true",
+ "properties": {
+ "large": {
+ "dynamic": "true",
+ "properties": {
+ "h": {
+ "type": "long"
+ },
+ "resize": {
+ "type": "string"
+ },
+ "w": {
+ "type": "long"
+ }
+ }
+ },
+ "medium": {
+ "dynamic": "true",
+ "properties": {
+ "h": {
+ "type": "long"
+ },
+ "resize": {
+ "type": "string"
+ },
+ "w": {
+ "type": "long"
+ }
+ }
+ },
+ "small": {
+ "dynamic": "true",
+ "properties": {
+ "h": {
+ "type": "long"
+ },
+ "resize": {
+ "type": "string"
+ },
+ "w": {
+ "type": "long"
+ }
+ }
+ },
+ "thumb": {
+ "dynamic": "true",
+ "properties": {
+ "h": {
+ "type": "long"
+ },
+ "resize": {
+ "type": "string"
+ },
+ "w": {
+ "type": "long"
+ }
+ }
+ }
+ }
+ },
+ "source_status_id": {
+ "type": "long"
+ },
+ "source_status_id_str": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ },
+ "mention_ids": {
+ "type": "long"
+ },
+ "mentions": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string"
+ }
+ ]
+ },
+ "source": {
+ "type": "string"
+ },
+ "text": {
+ "type": "string"
+ },
+ "user": {
+ "javaType": "com.datasift.twitter.User",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "followers_count": {
+ "type": "long"
+ },
+ "friends_count": {
+ "type": "long"
+ },
+ "geo_enabled": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "long"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "listed_count": {
+ "type": "long"
+ },
+ "location": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "profile_image_url": {
+ "type": "string"
+ },
+ "screen_name": {
+ "type": "string"
+ },
+ "statuses_count": {
+ "type": "long"
+ },
+ "time_zone": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ },
+ "utc_offset": {
+ "type": "long"
+ }
+ }
+ }
+ }
+ },
+ "retweeted": {
+ "javaType": "com.datasift.twitter.Retweeted",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "geo": {
+ "dynamic": "true",
+ "properties": {
+ "latitude": {
+ "type": "double"
+ },
+ "longitude": {
+ "type": "double"
+ }
+ }
+ },
+ "id": {
+ "type": "string"
+ },
+ "place": {
+ "dynamic": "true",
+ "properties": {
+ "attributes": {
+ "type": "object",
+ "dynamic": "true"
+ },
+ "country": {
+ "type": "string"
+ },
+ "country_code": {
+ "type": "string"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "place_type": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ },
+ "source": {
+ "type": "string"
+ },
+ "user": {
+ "dynamic": "true",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "followers_count": {
+ "type": "long"
+ },
+ "friends_count": {
+ "type": "long"
+ },
+ "geo_enabled": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "long"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "listed_count": {
+ "type": "long"
+ },
+ "location": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "profile_image_url": {
+ "type": "string"
+ },
+ "screen_name": {
+ "type": "string"
+ },
+ "statuses_count": {
+ "type": "long"
+ },
+ "time_zone": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ },
+ "utc_offset": {
+ "type": "long"
+ },
+ "verified": {
+ "type": "boolean"
+ }
+ }
+ }
+ }
+ },
+ "source": {
+ "type": "string"
+ },
+ "text": {
+ "type": "string"
+ },
+ "user": {
+ "javaType": "com.datasift.interaction.User",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "followers_count": {
+ "type": "long"
+ },
+ "friends_count": {
+ "type": "long"
+ },
+ "geo_enabled": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "long"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "listed_count": {
+ "type": "long"
+ },
+ "location": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "profile_image_url": {
+ "type": "string"
+ },
+ "screen_name": {
+ "type": "string"
+ },
+ "statuses_count": {
+ "type": "long"
+ },
+ "time_zone": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ },
+ "utc_offset": {
+ "type": "long"
+ },
+ "verified": {
+ "type": "boolean"
+ }
+ }
+ }
+ }
+ },
+ "youtube": {
+ "javaType": "com.datasift.youtube.YouTube",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "author": {
+ "javaType": "com.datasift.youtube.Author",
+ "type": "object",
+ "dynamic": "true",
+ "properties": {
+ "link": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ }
+ }
+ },
+ "commentslink": {
+ "type": "string"
+ },
+ "content": {
+ "type": "string"
+ },
+ "contenttype": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ },
+ "videolink": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
Added: incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
URL: http://svn.apache.org/viewvc/incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json?rev=1556893&view=auto
==============================================================================
--- incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json (added)
+++ incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json Thu Jan 9 17:27:50 2014
@@ -0,0 +1,21 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "com.datasift.DatasiftConfiguration",
+ "properties": {
+ "apiKey": {
+ "type": "string"
+ },
+ "userName": {
+ "type": "long"
+ },
+ "streamHash": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
\ No newline at end of file
Added: incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java
URL: http://svn.apache.org/viewvc/incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java?rev=1556893&view=auto
==============================================================================
--- incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java (added)
+++ incubator/streams/branches/sblackmon/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java Thu Jan 9 17:27:50 2014
@@ -0,0 +1,64 @@
+package com.datasift.test;
+
+import com.datasift.Datasift;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: sblackmon
+ * Date: 8/20/13
+ * Time: 5:57 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class DatasiftSerDeTest {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftSerDeTest.class);
+
+ private ObjectMapper mapper = new ObjectMapper();
+
+ @Ignore
+ @Test
+ public void Tests()
+ {
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
+ mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
+ mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
+
+ InputStream is = DatasiftSerDeTest.class.getResourceAsStream("/datasift_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ LOGGER.debug(line);
+
+ Datasift ser = mapper.readValue(line, Datasift.class);
+
+ String de = mapper.writeValueAsString(ser);
+
+ LOGGER.debug(de);
+
+ Datasift serde = mapper.readValue(de, Datasift.class);
+
+ Assert.assertEquals(ser, serde);
+
+ LOGGER.debug(mapper.writeValueAsString(serde));
+ }
+ } catch( Exception e ) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+}