You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/03/22 00:20:11 UTC
[33/71] [abbrv] fixing STREAMS-26 branch
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-persist-mongo/pom.xml
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-persist-mongo/pom.xml b/trunk/streams-contrib/streams-persist-mongo/pom.xml
deleted file mode 100644
index a209acb..0000000
--- a/trunk/streams-contrib/streams-persist-mongo/pom.xml
+++ /dev/null
@@ -1,92 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-mongo</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-java-driver</artifactId>
- <version>2.12.0-rc0</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.8</version>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jsonschema2pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <configuration>
- <addCompileSourceRoot>true</addCompileSourceRoot>
- <generateBuilders>true</generateBuilders>
- <sourcePaths>
- <sourcePath>src/main/jsonschema/org/apache/streams/mongo/MongoConfiguration.json</sourcePath>
- </sourcePaths>
- <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>org.apache.streams.mongo.pojo</targetPackage>
- <useLongIntegers>true</useLongIntegers>
- <useJodaDates>false</useJodaDates>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoConfigurator.java
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoConfigurator.java b/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoConfigurator.java
deleted file mode 100644
index cccdeb9..0000000
--- a/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoConfigurator.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package org.apache.streams.mongo;
-
-import com.google.common.base.Objects;
-import com.typesafe.config.Config;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Created by sblackmon on 12/10/13.
- */
-public class MongoConfigurator {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(MongoConfigurator.class);
-
- public static MongoConfiguration detectConfiguration(Config mongo) {
-
- MongoConfiguration mongoConfiguration = new MongoConfiguration();
-
- mongoConfiguration.setHost(mongo.getString("host"));
- mongoConfiguration.setPort(new Long(mongo.getInt("port")));
- mongoConfiguration.setDb(mongo.getString("db"));
- mongoConfiguration.setCollection(mongo.getString("collection"));
-
- if( mongo.hasPath("user"))
- mongoConfiguration.setUser(mongo.getString("user"));
- if( mongo.hasPath("password"))
- mongoConfiguration.setPassword(mongo.getString("password"));
- return mongoConfiguration;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriter.java
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriter.java b/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriter.java
deleted file mode 100644
index 5ab8470..0000000
--- a/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriter.java
+++ /dev/null
@@ -1,163 +0,0 @@
-package org.apache.streams.mongo;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Strings;
-import com.mongodb.DB;
-import com.mongodb.DBAddress;
-import com.mongodb.DBCollection;
-import com.mongodb.DBObject;
-import com.mongodb.MongoClient;
-import com.mongodb.util.JSON;
-import com.typesafe.config.Config;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-public class MongoPersistWriter implements StreamsPersistWriter, Runnable
-{
- private final static Logger LOGGER = LoggerFactory.getLogger(MongoPersistWriter.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = new ObjectMapper();
-
- private MongoConfiguration config;
-
- protected DB client;
- protected DBAddress dbaddress;
- protected DBCollection collection;
-
- protected List<DBObject> insertBatch = new ArrayList<DBObject>();
-
- public MongoPersistWriter() {
- Config config = StreamsConfigurator.config.getConfig("mongo");
- this.config = MongoConfigurator.detectConfiguration(config);
- this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
- }
-
- public MongoPersistWriter(Queue<StreamsDatum> persistQueue) {
- Config config = StreamsConfigurator.config.getConfig("mongo");
- this.config = MongoConfigurator.detectConfiguration(config);
- this.persistQueue = persistQueue;
- }
-
- private synchronized void connectToMongo()
- {
- try {
- dbaddress = new DBAddress(config.getHost(), config.getPort().intValue(), config.getDb());
- } catch (UnknownHostException e) {
- e.printStackTrace();
- return;
- }
-
- client = MongoClient.connect(dbaddress);
-
- if( !Strings.isNullOrEmpty(config.getUser()) && !Strings.isNullOrEmpty(config.getPassword()))
- client.authenticate(config.getUser(), config.getPassword().toCharArray());
-
- if( !client.collectionExists(config.getCollection())) {
- client.createCollection(config.getCollection(), null);
- };
-
- collection = client.getCollection(config.getCollection());
- }
-
- @Override
- public void write(StreamsDatum streamsDatum) {
-
- DBObject dbObject;
- if( streamsDatum.getDocument() instanceof String ) {
- dbObject = (DBObject) JSON.parse((String)streamsDatum.getDocument());
- } else {
- try {
- ObjectNode node = mapper.valueToTree(streamsDatum.getDocument());
- dbObject = (DBObject) JSON.parse(node.toString());
- } catch (Exception e) {
- e.printStackTrace();
- LOGGER.warn("Unsupported type: " + streamsDatum.getDocument().getClass());
- return;
- }
- }
-
- insertBatch.add(dbObject);
-
- if( insertBatch.size() % 100 == 0)
- try {
- flush();
- } catch (IOException e) {
- e.printStackTrace();
- return;
- }
- }
-
- public void flush() throws IOException
- {
- collection.insert(insertBatch);
- }
-
- public synchronized void close() throws IOException
- {
- client.cleanCursors(true);
- }
-
- @Override
- public void start() {
-
- connectToMongo();
-
- }
-
- @Override
- public void stop() {
-
- try {
- flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public void setPersistQueue(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
- }
-
- @Override
- public Queue<StreamsDatum> getPersistQueue() {
- return persistQueue;
- }
-
-
- @Override
- public void run() {
-
- start();
-
- Thread task = new Thread(new MongoPersistWriterTask(this));
- task.start();
-
- try {
- task.join();
- } catch (InterruptedException e) {
- e.printStackTrace();
- return;
- }
-
- stop();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriterTask.java b/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriterTask.java
deleted file mode 100644
index 398d1cd..0000000
--- a/trunk/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriterTask.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.apache.streams.mongo;
-
-import org.apache.streams.core.StreamsDatum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Random;
-
-public class MongoPersistWriterTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MongoPersistWriterTask.class);
-
- private MongoPersistWriter writer;
-
- public MongoPersistWriterTask(MongoPersistWriter writer) {
- this.writer = writer;
- }
-
- @Override
- public void run() {
-
- while(true) {
- if( writer.getPersistQueue().peek() != null ) {
- try {
- StreamsDatum entry = writer.persistQueue.remove();
- writer.write(entry);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- try {
- Thread.sleep(new Random().nextInt(1));
- } catch (InterruptedException e) {}
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-persist-mongo/src/main/jsonschema/org/apache/streams/mongo/MongoConfiguration.json
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-persist-mongo/src/main/jsonschema/org/apache/streams/mongo/MongoConfiguration.json b/trunk/streams-contrib/streams-persist-mongo/src/main/jsonschema/org/apache/streams/mongo/MongoConfiguration.json
deleted file mode 100644
index dc2a553..0000000
--- a/trunk/streams-contrib/streams-persist-mongo/src/main/jsonschema/org/apache/streams/mongo/MongoConfiguration.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "javaType" : "org.apache.streams.mongo.MongoConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "host": {
- "type": "string",
- "description": "Hbase host"
- },
- "port": {
- "type": "integer",
- "description": "ZK Port"
- },
- "user": {
- "type": "string",
- "description": "User"
- },
- "password": {
- "type": "string",
- "description": "Password"
- },
- "db": {
- "type": "string",
- "description": "DB"
- },
- "collection": {
- "type": "string",
- "description": "Collection"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-persist-mongo/src/main/resources/reference.properties
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-persist-mongo/src/main/resources/reference.properties b/trunk/streams-contrib/streams-persist-mongo/src/main/resources/reference.properties
deleted file mode 100644
index 699f655..0000000
--- a/trunk/streams-contrib/streams-persist-mongo/src/main/resources/reference.properties
+++ /dev/null
@@ -1,10 +0,0 @@
-hbase.rootdir = "hdfs://localhost:8020/hbase"
-
-zookeeper.znode.parent = "/hbase"
-
-zookeeper.znode.rootserver = "localhost"
-
-hbase.zookeeper.quorum = "localhost"
-
-hbase.zookeeper.property.clientPort = 2181
-
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-provider-datasift/pom.xml
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-provider-datasift/pom.xml b/trunk/streams-contrib/streams-provider-datasift/pom.xml
deleted file mode 100644
index b49db93..0000000
--- a/trunk/streams-contrib/streams-provider-datasift/pom.xml
+++ /dev/null
@@ -1,95 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-provider-datasift</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.8</version>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jsonschema2pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <configuration>
- <addCompileSourceRoot>true</addCompileSourceRoot>
- <generateBuilders>true</generateBuilders>
- <sourcePaths>
- <sourcePath>src/main/jsonschema/com/datasift/Datasift.json</sourcePath>
- <sourcePath>src/main/jsonschema/com/datasift/DatasiftConfiguration.json</sourcePath>
- </sourcePaths>
- <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>com.datasift</targetPackage>
- <useLongIntegers>true</useLongIntegers>
- <useJodaDates>false</useJodaDates>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json b/trunk/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
deleted file mode 100644
index d6c973d..0000000
--- a/trunk/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/Datasift.json
+++ /dev/null
@@ -1,1205 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "javaType": "com.datasift.Datasift",
- "properties": {
- "blog": {
- "javaType": "com.datasift.blog.Blog",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "type": "object",
- "javaType": "com.datasift.blog.BlogAuthor",
- "dynamic": "true",
- "properties": {
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string"
- },
- "domain": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- },
- "demographic": {
- "dynamic": "true",
- "properties": {
- "gender": {
- "type": "string"
- }
- }
- },
- "facebook": {
- "javaType": "com.datasift.config.Facebook",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "application": {
- "type": "string"
- },
- "author": {
- "javaType": "com.datasift.config.Author",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "avatar": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "created_at": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "likes": {
- "javaType": "com.datasift.config.Likes",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "count": {
- "type": "long"
- },
- "ids": {
- "type": "string"
- },
- "names": {
- "type": "string"
- }
- }
- },
- "message": {
- "type": "string"
- },
- "source": {
- "type": "string"
- },
- "to": {
- "javaType": "com.datasift.config.To",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "ids": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "names": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "type": {
- "type": "string"
- }
- }
- },
- "interaction": {
- "javaType": "com.datasift.interaction.Interaction",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "javaType": "com.datasift.interaction.Author",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "avatar": {
- "type": "string"
- },
- "id": {
- "type": "long"
- },
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "username": {
- "type": "string"
- }
- }
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string"
- },
- "geo": {
- "javaType": "com.datasift.interaction.Geo",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "latitude": {
- "type": "double"
- },
- "longitude": {
- "type": "double"
- }
- }
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "schema": {
- "dynamic": "true",
- "properties": {
- "version": {
- "type": "long"
- }
- }
- },
- "source": {
- "type": "string"
- },
- "tags": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- },
- "klout": {
- "javaType": "com.datasift.klout.Klout",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "score": {
- "type": "long"
- }
- }
- },
- "language": {
- "javaType": "com.datasift.interaction.Language",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "confidence": {
- "type": "long"
- },
- "tag": {
- "type": "string"
- }
- }
- },
- "links": {
- "javaType": "com.datasift.interaction.Links",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "code": {
- "type": "long"
- },
- "created_at": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "hops": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "meta": {
- "dynamic": "true",
- "javaType": "com.datasift.interaction.Meta",
- "type": "object",
- "properties": {
- "charset": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "content_type": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "description": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "keywords": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "lang": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "opengraph": {
- "type": "array",
- "items": [
- {
- "javaType": "com.datasift.interaction.meta.OpenGraph",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "description": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "site_name": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- },
- "twitter": {
- "type": "array",
- "items": [
- {
- "javaType": "com.datasift.interaction.meta.Twitter",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "card": {
- "type": "string"
- },
- "creator": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "player": {
- "type": "string"
- },
- "player_height": {
- "type": "string"
- },
- "player_width": {
- "type": "string"
- },
- "site": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- }
- }
- },
- "normalized_url": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "retweet_count": {
- "type": "long"
- },
- "title": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "url": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "salience": {
- "javaType": "com.datasift.salience.Salience",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "content": {
- "javaType": "com.datasift.salience.content.Content",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "entities": {
- "type": "array",
- "items": [
- {
- "javaType": "com.datasift.salience.content.Entities",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "about": {
- "type": "long"
- },
- "confident": {
- "type": "long"
- },
- "evidence": {
- "type": "long"
- },
- "label": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "sentiment": {
- "type": "long"
- },
- "themes": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- }
- ]
- },
- "sentiment": {
- "type": "long"
- }
- }
- },
- "title": {
- "javaType": "com.datasift.salience.title.Title",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "entities": {
- "type": "array",
- "items": [
- {
- "javaType": "com.datasift.salience.title.Entities",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "about": {
- "type": "long"
- },
- "confident": {
- "type": "long"
- },
- "evidence": {
- "type": "long"
- },
- "label": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "sentiment": {
- "type": "long"
- },
- "themes": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- }
- ]
- },
- "sentiment": {
- "type": "long"
- }
- }
- }
- }
- },
- "trends": {
- "javaType": "com.datasift.trends.Trends",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "content": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "source": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "type": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "twitter": {
- "javaType": "com.datasift.twitter.Twitter",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "created_at": {
- "type": "string"
- },
- "domains": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "filter_level": {
- "type": "string"
- },
- "geo": {
- "javaType": "com.datasift.twitter.Geo",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "latitude": {
- "type": "double"
- },
- "longitude": {
- "type": "double"
- }
- }
- },
- "hashtags": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "id": {
- "type": "string"
- },
- "in_reply_to_screen_name": {
- "type": "string"
- },
- "in_reply_to_status_id": {
- "type": "string"
- },
- "in_reply_to_user_id": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "links": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "media": {
- "type": "array",
- "items": [
- {
- "javaType": "com.datasift.twitter.Media",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "display_url": {
- "type": "string"
- },
- "expanded_url": {
- "type": "string"
- },
- "id": {
- "type": "long"
- },
- "id_str": {
- "type": "string"
- },
- "media_url": {
- "type": "string"
- },
- "media_url_https": {
- "type": "string"
- },
- "sizes": {
- "dynamic": "true",
- "properties": {
- "large": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "long"
- }
- }
- },
- "medium": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "long"
- }
- }
- },
- "small": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "long"
- }
- }
- },
- "thumb": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "long"
- }
- }
- }
- }
- },
- "source_status_id": {
- "type": "long"
- },
- "source_status_id_str": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- },
- "mention_ids": {
- "type": "long"
- },
- "mentions": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "place": {
- "javaType": "com.datasift.twitter.Place",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "attributes": {
- "dynamic": "true",
- "properties": {
- "street_address": {
- "type": "string"
- }
- }
- },
- "country": {
- "type": "string"
- },
- "country_code": {
- "type": "string"
- },
- "full_name": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "place_type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "retweet": {
- "javaType": "com.datasift.twitter.Retweet",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "count": {
- "type": "long"
- },
- "created_at": {
- "type": "string"
- },
- "domains": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "hashtags": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "id": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "links": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "media": {
- "dynamic": "true",
- "properties": {
- "display_url": {
- "type": "string"
- },
- "expanded_url": {
- "type": "string"
- },
- "id": {
- "type": "long"
- },
- "id_str": {
- "type": "string"
- },
- "media_url": {
- "type": "string"
- },
- "media_url_https": {
- "type": "string"
- },
- "sizes": {
- "dynamic": "true",
- "properties": {
- "large": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "long"
- }
- }
- },
- "medium": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "long"
- }
- }
- },
- "small": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "long"
- }
- }
- },
- "thumb": {
- "dynamic": "true",
- "properties": {
- "h": {
- "type": "long"
- },
- "resize": {
- "type": "string"
- },
- "w": {
- "type": "long"
- }
- }
- }
- }
- },
- "source_status_id": {
- "type": "long"
- },
- "source_status_id_str": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "mention_ids": {
- "type": "long"
- },
- "mentions": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "source": {
- "type": "string"
- },
- "text": {
- "type": "string"
- },
- "user": {
- "javaType": "com.datasift.twitter.User",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "created_at": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "followers_count": {
- "type": "long"
- },
- "friends_count": {
- "type": "long"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "id": {
- "type": "long"
- },
- "id_str": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "listed_count": {
- "type": "long"
- },
- "location": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "profile_image_url": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "statuses_count": {
- "type": "long"
- },
- "time_zone": {
- "type": "string"
- },
- "url": {
- "type": "string"
- },
- "utc_offset": {
- "type": "long"
- }
- }
- }
- }
- },
- "retweeted": {
- "javaType": "com.datasift.twitter.Retweeted",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "created_at": {
- "type": "string"
- },
- "geo": {
- "dynamic": "true",
- "properties": {
- "latitude": {
- "type": "double"
- },
- "longitude": {
- "type": "double"
- }
- }
- },
- "id": {
- "type": "string"
- },
- "place": {
- "dynamic": "true",
- "properties": {
- "attributes": {
- "type": "object",
- "dynamic": "true"
- },
- "country": {
- "type": "string"
- },
- "country_code": {
- "type": "string"
- },
- "full_name": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "place_type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "source": {
- "type": "string"
- },
- "user": {
- "dynamic": "true",
- "properties": {
- "created_at": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "followers_count": {
- "type": "long"
- },
- "friends_count": {
- "type": "long"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "id": {
- "type": "long"
- },
- "id_str": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "listed_count": {
- "type": "long"
- },
- "location": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "profile_image_url": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "statuses_count": {
- "type": "long"
- },
- "time_zone": {
- "type": "string"
- },
- "url": {
- "type": "string"
- },
- "utc_offset": {
- "type": "long"
- },
- "verified": {
- "type": "boolean"
- }
- }
- }
- }
- },
- "source": {
- "type": "string"
- },
- "text": {
- "type": "string"
- },
- "user": {
- "javaType": "com.datasift.interaction.User",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "created_at": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "followers_count": {
- "type": "long"
- },
- "friends_count": {
- "type": "long"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "id": {
- "type": "long"
- },
- "id_str": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "listed_count": {
- "type": "long"
- },
- "location": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "profile_image_url": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "statuses_count": {
- "type": "long"
- },
- "time_zone": {
- "type": "string"
- },
- "url": {
- "type": "string"
- },
- "utc_offset": {
- "type": "long"
- },
- "verified": {
- "type": "boolean"
- }
- }
- }
- }
- },
- "youtube": {
- "javaType": "com.datasift.youtube.YouTube",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "javaType": "com.datasift.youtube.Author",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "commentslink": {
- "type": "string"
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "videolink": {
- "type": "string"
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json b/trunk/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
deleted file mode 100644
index 531f30b..0000000
--- a/trunk/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "javaType" : "com.datasift.DatasiftConfiguration",
- "properties": {
- "apiKey": {
- "type": "string"
- },
- "userName": {
- "type": "long"
- },
- "streamHash": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6ea69f29/trunk/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java
----------------------------------------------------------------------
diff --git a/trunk/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java b/trunk/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java
deleted file mode 100644
index 6a586d0..0000000
--- a/trunk/streams-contrib/streams-provider-datasift/src/test/java/com/datasift/test/DatasiftSerDeTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package com.datasift.test;
-
-import com.datasift.Datasift;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Created with IntelliJ IDEA.
- * User: sblackmon
- * Date: 8/20/13
- * Time: 5:57 PM
- * To change this template use File | Settings | File Templates.
- */
-public class DatasiftSerDeTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftSerDeTest.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- @Ignore
- @Test
- public void Tests()
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = DatasiftSerDeTest.class.getResourceAsStream("/datasift_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- Datasift ser = mapper.readValue(line, Datasift.class);
-
- String de = mapper.writeValueAsString(ser);
-
- LOGGER.debug(de);
-
- Datasift serde = mapper.readValue(de, Datasift.class);
-
- Assert.assertEquals(ser, serde);
-
- LOGGER.debug(mapper.writeValueAsString(serde));
- }
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}