You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/11/08 21:12:52 UTC
[01/38] incubator-streams git commit: just pons
Repository: incubator-streams
Updated Branches:
refs/heads/STREAMS-49 85e0b5007 -> 3026b8319
just pons
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/1464819f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/1464819f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/1464819f
Branch: refs/heads/STREAMS-49
Commit: 1464819fd7ea147830a4211ea2ac6af4aa715022
Parents: 35a8fbf
Author: sblackmon <sb...@apache.org>
Authored: Sun Sep 14 10:24:47 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Sun Sep 14 10:24:47 2014 -0500
----------------------------------------------------------------------
streams-components/pom.xml | 62 ++++++++++
.../streams-processor-http/pom.xml | 114 +++++++++++++++++++
2 files changed, 176 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1464819f/streams-components/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/pom.xml b/streams-components/pom.xml
new file mode 100644
index 0000000..26384b1
--- /dev/null
+++ b/streams-components/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streams-project</artifactId>
+ <groupId>org.apache.streams</groupId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streams-components</artifactId>
+
+ <packaging>pom</packaging>
+ <name>streams-components</name>
+
+ <properties>
+
+ </properties>
+
+ <modules>
+ <module>streams-processor-http</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1464819f/streams-components/streams-processor-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/pom.xml b/streams-components/streams-processor-http/pom.xml
new file mode 100644
index 0000000..67538aa
--- /dev/null
+++ b/streams-components/streams-processor-http/pom.xml
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-project</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>streams-processor-http</artifactId>
+
+ <name>streams-processor-http</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-core</artifactId>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-source-jaxb2</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jaxb2</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>org.apache.streams.http</targetPackage>
+ <useLongIntegers>true</useLongIntegers>
+ <useJodaDates>true</useJodaDates>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
[05/38] incubator-streams git commit: Writer can now write to
us-west-1 Simplified configurator
Posted by sb...@apache.org.
Writer can now write to us-west-1
Simplified configurator
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/25af3ad3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/25af3ad3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/25af3ad3
Branch: refs/heads/STREAMS-49
Commit: 25af3ad3dadf40d57eeb62479b3e73cd5170275e
Parents: 1788691
Author: sblackmon <sb...@apache.org>
Authored: Tue Oct 7 13:44:22 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Oct 7 13:44:22 2014 -0500
----------------------------------------------------------------------
streams-contrib/streams-amazon-aws/pom.xml | 2 +-
.../org/apache/streams/s3/S3Configurator.java | 55 +++++++++-----------
.../org/apache/streams/s3/S3PersistReader.java | 11 ++--
.../org/apache/streams/s3/S3PersistWriter.java | 10 ++--
.../org/apache/streams/s3/S3Configuration.json | 18 +++++--
5 files changed, 55 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/25af3ad3/streams-contrib/streams-amazon-aws/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-amazon-aws/pom.xml b/streams-contrib/streams-amazon-aws/pom.xml
index 57a67cb..c9b73ae 100644
--- a/streams-contrib/streams-amazon-aws/pom.xml
+++ b/streams-contrib/streams-amazon-aws/pom.xml
@@ -45,7 +45,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
- <version>1.7.5</version>
+ <version>1.8.11</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/25af3ad3/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3Configurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3Configurator.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3Configurator.java
index dfa0426..6bf1672 100644
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3Configurator.java
+++ b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3Configurator.java
@@ -18,7 +18,9 @@
package org.apache.streams.s3;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
import com.typesafe.config.Config;
+import com.typesafe.config.ConfigRenderOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,53 +32,46 @@ public class S3Configurator {
public static S3Configuration detectConfiguration(Config s3) {
- S3Configuration s3Configuration = new S3Configuration();
+ S3Configuration s3Configuration = null;
- s3Configuration.setBucket(s3.getString("bucket"));
- s3Configuration.setKey(s3.getString("key"));
- s3Configuration.setSecretKey(s3.getString("secretKey"));
-
- // The Amazon S3 Library defaults to HTTPS
- String protocol = (!s3.hasPath("protocol") ? "https": s3.getString("protocol")).toLowerCase();
-
- if(!(protocol.equals("https") || protocol.equals("http"))) {
- // you must specify either HTTP or HTTPS
- throw new RuntimeException("You must specify either HTTP or HTTPS as a protocol");
+ try {
+ s3Configuration = mapper.readValue(s3.root().render(ConfigRenderOptions.concise()), S3Configuration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse S3Configuration");
}
- s3Configuration.setProtocol(protocol.toLowerCase());
-
return s3Configuration;
}
public static S3ReaderConfiguration detectReaderConfiguration(Config s3) {
- S3Configuration S3Configuration = detectConfiguration(s3);
- S3ReaderConfiguration s3ReaderConfiguration = mapper.convertValue(S3Configuration, S3ReaderConfiguration.class);
+ S3ReaderConfiguration s3Configuration = null;
- s3ReaderConfiguration.setReaderPath(s3.getString("readerPath"));
+ try {
+ s3Configuration = mapper.readValue(s3.root().render(ConfigRenderOptions.concise()), S3ReaderConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse S3Configuration");
+ }
- return s3ReaderConfiguration;
+ return s3Configuration;
}
public static S3WriterConfiguration detectWriterConfiguration(Config s3) {
- S3Configuration s3Configuration = detectConfiguration(s3);
- S3WriterConfiguration s3WriterConfiguration = mapper.convertValue(s3Configuration, S3WriterConfiguration.class);
+ S3WriterConfiguration s3Configuration = null;
- String rootPath = s3.getString("writerPath");
-
- // if the root path doesn't end in a '/' then we need to force the '/' at the end of the path.
- s3WriterConfiguration.setWriterPath(rootPath + (rootPath.endsWith("/") ? "" : "/"));
-
- s3WriterConfiguration.setWriterFilePrefix(s3.hasPath("writerFilePrefix") ? s3.getString("writerFilePrefix") : "default");
+ try {
+ s3Configuration = mapper.readValue(s3.root().render(ConfigRenderOptions.concise()), S3WriterConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse S3Configuration");
+ }
- if(s3.hasPath("maxFileSize"))
- s3WriterConfiguration.setMaxFileSize((long)s3.getInt("maxFileSize"));
- if(s3.hasPath("chunk"))
- s3WriterConfiguration.setChunk(s3.getBoolean("chunk"));
+ Preconditions.checkArgument(s3Configuration.getWriterPath().endsWith("/"), s3Configuration.getWriterPath() + " must end with '/'");
- return s3WriterConfiguration;
+ return s3Configuration;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/25af3ad3/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
index 4f62a06..5709f22 100644
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
+++ b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
@@ -21,12 +21,15 @@ import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Strings;
import com.google.common.collect.Queues;
import org.apache.streams.core.*;
import org.joda.time.DateTime;
@@ -104,13 +107,15 @@ public class S3PersistReader implements StreamsPersistReader, DatumStatusCountab
AWSCredentials credentials = new BasicAWSCredentials(s3ReaderConfiguration.getKey(), s3ReaderConfiguration.getSecretKey());
ClientConfiguration clientConfig = new ClientConfiguration();
- clientConfig.setProtocol(Protocol.valueOf(s3ReaderConfiguration.getProtocol().toUpperCase()));
+ clientConfig.setProtocol(Protocol.valueOf(s3ReaderConfiguration.getProtocol().toString()));
- // We want path style access
+ // We do not want path style access
S3ClientOptions clientOptions = new S3ClientOptions();
- clientOptions.setPathStyleAccess(true);
+ clientOptions.setPathStyleAccess(false);
this.amazonS3Client = new AmazonS3Client(credentials, clientConfig);
+ if( !Strings.isNullOrEmpty(s3ReaderConfiguration.getRegion()))
+ this.amazonS3Client.setRegion(Region.getRegion(Regions.fromName(s3ReaderConfiguration.getRegion())));
this.amazonS3Client.setS3ClientOptions(clientOptions);
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/25af3ad3/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistWriter.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistWriter.java
index 058f748..9111265 100644
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistWriter.java
+++ b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistWriter.java
@@ -21,6 +21,8 @@ import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -256,13 +258,15 @@ public class S3PersistWriter implements StreamsPersistWriter, DatumStatusCountab
AWSCredentials credentials = new BasicAWSCredentials(s3WriterConfiguration.getKey(), s3WriterConfiguration.getSecretKey());
ClientConfiguration clientConfig = new ClientConfiguration();
- clientConfig.setProtocol(Protocol.valueOf(s3WriterConfiguration.getProtocol().toUpperCase()));
+ clientConfig.setProtocol(Protocol.valueOf(s3WriterConfiguration.getProtocol().toString()));
- // We want path style access
+ // We do not want path style access
S3ClientOptions clientOptions = new S3ClientOptions();
- clientOptions.setPathStyleAccess(true);
+ clientOptions.setPathStyleAccess(false);
this.amazonS3Client = new AmazonS3Client(credentials, clientConfig);
+ if( !Strings.isNullOrEmpty(s3WriterConfiguration.getRegion()))
+ this.amazonS3Client.setRegion(Region.getRegion(Regions.fromName(s3WriterConfiguration.getRegion())));
this.amazonS3Client.setS3ClientOptions(clientOptions);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/25af3ad3/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
index 863668f..36e89d0 100644
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
+++ b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
@@ -7,19 +7,29 @@
"properties": {
"key": {
"type": "string",
- "description": "Your Amazon Key"
+ "description": "Your Amazon Key",
+ "required": true
},
"secretKey": {
"type": "string",
- "description": "Your Amazon Secret Key"
+ "description": "Your Amazon Secret Key",
+ "required": true
},
"bucket": {
"type": "string",
- "description": "The AWS bucket you want to write to"
+ "description": "Your AWS bucket",
+ "required": true
},
"protocol": {
"type": "string",
- "description": "Whether you are using HTTP or HTTPS"
+ "description": "Whether you are using HTTP or HTTPS",
+ "enum": ["HTTP", "HTTPS"],
+ "default": "HTTPS"
+ },
+ "region": {
+ "type": "string",
+ "description": "The AWS region where your bucket resides",
+ "required": true
}
}
}
\ No newline at end of file
[02/38] incubator-streams git commit: added streams-components module
added streams-processor-http module activated streams-pojo-extensions module
tweaks to pojo, require id and remove non-sensical defaults
Posted by sb...@apache.org.
added streams-components module
added streams-processor-http module
activated streams-pojo-extensions module
tweaks to pojo, require id and remove non-sensical defaults
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/34232ad8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/34232ad8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/34232ad8
Branch: refs/heads/STREAMS-49
Commit: 34232ad87b50253913fba0dabb6fa9af591f388d
Parents: 1464819
Author: sblackmon <sb...@apache.org>
Authored: Mon Sep 15 13:50:32 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Mon Sep 15 13:50:32 2014 -0500
----------------------------------------------------------------------
pom.xml | 2 +
.../streams-processor-http/README.md | 16 ++
.../streams-processor-http/pom.xml | 42 +++-
.../components/http/SimpleHTTPGetProcessor.java | 213 +++++++++++++++++++
.../HttpProcessorConfiguration.json | 47 ++++
.../api/FacebookPostActivitySerializer.java | 1 -
.../streams-provider-twitter/pom.xml | 5 +
.../processor/TwitterUrlApiProcessor.java | 44 ++++
streams-pojo-extensions/pom.xml | 64 ++++++
.../apache/streams/data/util/ExtensionUtil.java | 94 ++++++++
.../apache/streams/data/util/ActivityUtil.java | 14 +-
.../org/apache/streams/pojo/json/activity.json | 3 +-
.../org/apache/streams/pojo/json/object.json | 2 +-
13 files changed, 537 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c2e1766..59fa634 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,7 +96,9 @@
<module>streams-osgi-components</module>
<module>streams-core</module>
<module>streams-config</module>
+ <module>streams-components</module>
<module>streams-pojo</module>
+ <module>streams-pojo-extensions</module>
<module>streams-util</module>
<module>streams-contrib</module>
<module>streams-runtimes</module>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-components/streams-processor-http/README.md
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/README.md b/streams-components/streams-processor-http/README.md
new file mode 100644
index 0000000..62dd4c1
--- /dev/null
+++ b/streams-components/streams-processor-http/README.md
@@ -0,0 +1,16 @@
+streams-processor-http
+=====================
+
+Hit an http endpoint and place the result in extensions
+
+Example SimpleHTTPGetProcessor configuration:
+
+ "http": {
+ "protocol": "http",
+ "hostname": "urls.api.twitter.com",
+ "port": 9300,
+ "resourceUri": "1/urls/count.json"
+ }
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-components/streams-processor-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/pom.xml b/streams-components/streams-processor-http/pom.xml
index 67538aa..d9215ad 100644
--- a/streams-components/streams-processor-http/pom.xml
+++ b/streams-components/streams-processor-http/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-project</artifactId>
+ <artifactId>streams-components</artifactId>
<version>0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -40,6 +40,45 @@
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo-extensions</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.3.5</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
</dependencies>
<build>
@@ -100,6 +139,7 @@
<targetPackage>org.apache.streams.http</targetPackage>
<useLongIntegers>true</useLongIntegers>
<useJodaDates>true</useJodaDates>
+ <includeJsr303Annotations>true</includeJsr303Annotations>
</configuration>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
new file mode 100644
index 0000000..d76d839
--- /dev/null
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
@@ -0,0 +1,213 @@
+package org.apache.streams.components.http;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigValue;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.data.util.ActivityUtil;
+import org.apache.streams.data.util.ExtensionUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.Validation;
+import javax.validation.ValidatorFactory;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
+
+ private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
+
+ // from root config id
+ private final static String EXTENSION = "account_type";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPGetProcessor.class);
+
+ protected ObjectMapper mapper;
+
+ protected URIBuilder uriBuilder;
+
+ protected CloseableHttpClient httpclient;
+
+ protected HttpProcessorConfiguration configuration;
+//
+// // authorized only
+// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
+// //private String authHeader;
+//
+ public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
+ LOGGER.info("creating SimpleHTTPGetProcessor");
+ LOGGER.info(processorConfiguration.toString());
+ this.configuration = processorConfiguration;
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+
+ return Maps.newHashMap();
+ }
+
+ /**
+ Override this to store a result other than exact json representation of response
+ */
+ protected ObjectNode prepareExtensionFragment(String entityString) {
+
+ try {
+ return mapper.readValue(entityString, ObjectNode.class);
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ }
+ }
+
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode getRootDocument(StreamsDatum datum) {
+
+ try {
+ String json = datum.getDocument() instanceof String ?
+ (String) datum.getDocument() :
+ mapper.writeValueAsString(datum.getDocument());
+ return mapper.readValue(json, ObjectNode.class);
+ } catch (JsonProcessingException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ }
+
+ }
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode getEntityToExtend(ObjectNode rootDocument) {
+
+ if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
+ return rootDocument;
+ else
+ return (ObjectNode) rootDocument.get(this.configuration.getEntity().toString());
+
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+
+ List<StreamsDatum> result = Lists.newArrayList();
+
+ ObjectNode rootDocument = getRootDocument(entry);
+
+ Map<String, String> params = prepareParams(entry);
+
+ URI uri;
+ for( Map.Entry<String,String> param : params.entrySet()) {
+ uriBuilder = uriBuilder.setParameter(param.getKey(), param.getValue());
+ }
+ try {
+ uri = uriBuilder.build();
+ } catch (URISyntaxException e) {
+ LOGGER.error("URI error {}", uriBuilder.toString());
+ return result;
+ }
+
+ HttpGet httpget = new HttpGet(uri);
+ httpget.addHeader("content-type", this.configuration.getContentType());
+ //httpget.addHeader("Authorization", String.format("Basic %s", authHeader));
+
+ CloseableHttpResponse response = null;
+
+ String entityString = null;
+ try {
+ response = httpclient.execute(httpget);
+ HttpEntity entity = response.getEntity();
+ // TODO: handle rate-limiting
+ if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
+ entityString = EntityUtils.toString(entity);
+ }
+ } catch (IOException e) {
+ LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, e.getMessage());
+ return result;
+ } finally {
+ try {
+ response.close();
+ } catch (IOException e) {}
+ try {
+ httpclient.close();
+ } catch (IOException e) {}
+ }
+
+ if( entityString == null )
+ return result;
+
+ LOGGER.debug(entityString);
+
+ ObjectNode extensionFragment = prepareExtensionFragment(entityString);
+
+ ObjectNode extensionEntity = getEntityToExtend(rootDocument);
+
+ ExtensionUtil.ensureExtensions(extensionEntity);
+
+ ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
+
+ entry.setDocument(extensionEntity);
+
+ result.add(entry);
+
+ return result;
+
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
+ Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
+
+ mapper = StreamsJacksonMapper.getInstance();
+
+ uriBuilder = new URIBuilder()
+ .setScheme(this.configuration.getProtocol())
+ .setHost(this.configuration.getHostname())
+ .setPath(this.configuration.getResourceUri());
+
+ httpclient = HttpClients.createDefault();
+ // StringBuilder stringBuilder = new StringBuilder();
+// stringBuilder.append(peoplePatternConfiguration.getUsername());
+// stringBuilder.append(":");
+// stringBuilder.append(peoplePatternConfiguration.getPassword());
+// String string = stringBuilder.toString();
+// authHeader = Base64.encodeBase64String(string.getBytes());
+ }
+
+ @Override
+ public void cleanUp() {
+ LOGGER.info("shutting down SimpleHTTPGetProcessor");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json b/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
new file mode 100644
index 0000000..40c3bcd
--- /dev/null
+++ b/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
@@ -0,0 +1,47 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.components.http.HttpProcessorConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "protocol": {
+ "type": "string",
+ "description": "Protocol",
+ "default": "http"
+ },
+ "hostname": {
+ "type": "string",
+ "description": "Hostname",
+ "required" : true
+ },
+ "port": {
+ "type": "integer",
+ "description": "Port",
+ "default": 80
+ },
+ "resourceUri": {
+ "type": "string",
+ "description": "Resource URI",
+ "required" : true
+ },
+ "content-type": {
+ "type": "string",
+ "description": "Resource URI",
+ "required" : true,
+ "default": "application/json"
+ },
+ "entity": {
+ "type": "string",
+ "description": "Entity to extend",
+ "enum": [ "activity", "actor", "object", "target" ],
+ "required" : true,
+ "default": "activity"
+ },
+ "extension": {
+ "type": "string",
+ "description": "Extension identifier",
+ "required" : true
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
index aa718fb..de39262 100644
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
+++ b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
@@ -78,7 +78,6 @@ public class FacebookPostActivitySerializer implements ActivitySerializer<org.ap
setProvider(activity);
setObjectType(post.getType(), activity);
parseObject(activity, mapper.convertValue(post, ObjectNode.class));
- fixObjectId(activity);
fixContentFromSummary(activity);
activity.setVerb("post");
List<String> links = Lists.newLinkedList();
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
index 79d1608..f0d65f8 100644
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@ -54,6 +54,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-processor-http</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
new file mode 100644
index 0000000..77965c4
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
@@ -0,0 +1,44 @@
+package org.apache.streams.twitter.processor;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import org.apache.streams.components.http.HttpProcessorConfiguration;
+import org.apache.streams.components.http.SimpleHTTPGetProcessor;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.pojo.json.Activity;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by sblackmon on 9/14/14.
+ */
+public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements StreamsProcessor {
+
+ public TwitterUrlApiProcessor(HttpProcessorConfiguration processorConfiguration) {
+ super(processorConfiguration);
+ this.configuration.setHostname("urls.api.twitter.com");
+ this.configuration.setResourceUri("/1/urls/count.json");
+ this.configuration.setExtension("twitter_url_count");
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ Preconditions.checkArgument(entry.getDocument() instanceof Activity);
+ Activity activity = mapper.convertValue(entry, Activity.class);
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(activity.getUrl()));
+ return super.process(entry);
+ }
+
+ @Override
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+
+ Map<String, String> params = Maps.newHashMap();
+
+ params.put("url", mapper.convertValue(entry, Activity.class).getUrl());
+
+ return params;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo-extensions/pom.xml
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/pom.xml b/streams-pojo-extensions/pom.xml
new file mode 100644
index 0000000..7f3f1a5
--- /dev/null
+++ b/streams-pojo-extensions/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-project</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>streams-pojo-extensions</artifactId>
+
+ <name>streams-pojo-extensions</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
new file mode 100644
index 0000000..a8d068a
--- /dev/null
+++ b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
@@ -0,0 +1,94 @@
+package org.apache.streams.data.util;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Maps;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.ActivityObject;
+
+import java.util.Map;
+
+public class ExtensionUtil {
+
+ /**
+ * Property on the activity object to use for extensions
+ */
+ public static final String EXTENSION_PROPERTY = "extensions";
+ /**
+ * The number of +1, Like, favorites, etc that the post has received
+ */
+ public static final String LIKES_EXTENSION = "likes";
+ /**
+ * The number of retweets, shares, etc that the post has received
+ */
+ public static final String REBROADCAST_EXTENSION = "rebroadcasts";
+ /**
+ * The language of the post
+ */
+ public static final String LANGUAGE_EXTENSION = "language";
+ /**
+ * Location that the post was made or the actor's residence
+ */
+ public static final String LOCATION_EXTENSION = "location";
+ /**
+ * Country that the post was made
+ */
+ public static final String LOCATION_EXTENSION_COUNTRY = "country";
+ /**
+ * Specific JSON-geo coordinates (long,lat)
+ */
+ public static final String LOCATION_EXTENSION_COORDINATES = "coordinates";
+
+ private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+
+ public static Map<String, Object> getExtensions(ObjectNode object) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ return extensions;
+ }
+
+ public static Object getExtension(ObjectNode object, String key) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ return extensions.get(key);
+ }
+
+ public static void setExtensions(ObjectNode object, Map<String, Object> extensions) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ activityObject.setAdditionalProperty(EXTENSION_PROPERTY, extensions);
+ };
+
+ public static void addExtension(ObjectNode object, String key, Object extension) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ extensions.put(key, extension);
+ };
+
+ public static void addExtensions(ObjectNode object, Map<String, Object> extensions) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ for( Map.Entry<String, Object> item : extensions.entrySet())
+ activityObject.getAdditionalProperties().put(item.getKey(), item.getValue());
+ };
+
+ public static void removeExtension(ObjectNode object, String key) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ extensions.remove(key);
+ };
+
+ /**
+ * Creates a standard extension property
+ * @param object objectnode to create the property in
+ * @return the Map representing the extensions property
+ */
+ @SuppressWarnings("unchecked")
+ public static Map<String, Object> ensureExtensions(ObjectNode object) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ if(extensions == null) {
+ extensions = Maps.newHashMap();
+ setExtensions(object, extensions);
+ }
+ return getExtensions(object);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java b/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
index 3684b32..04ee923 100644
--- a/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
+++ b/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
@@ -19,6 +19,7 @@
package org.apache.streams.data.util;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.Activity;
@@ -61,7 +62,7 @@ public class ActivityUtil {
*/
public static final String LOCATION_EXTENSION_COORDINATES = "coordinates";
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+ private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
/**
* Creates a standard extension property
@@ -69,13 +70,14 @@ public class ActivityUtil {
* @return the Map representing the extensions property
*/
@SuppressWarnings("unchecked")
+ @Deprecated
public static Map<String, Object> ensureExtensions(Activity activity) {
- Map<String, Object> properties = (Map)activity.getAdditionalProperties().get(EXTENSION_PROPERTY);
- if(properties == null) {
- properties = new HashMap<String, Object>();
- activity.setAdditionalProperty(EXTENSION_PROPERTY, properties);
+ Map<String, Object> extensions = (Map)activity.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ if(extensions == null) {
+ extensions = new HashMap<String, Object>();
+ activity.setAdditionalProperty(EXTENSION_PROPERTY, extensions);
}
- return properties;
+ return extensions;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json b/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json
index 45c2276..a68ce00 100644
--- a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json
+++ b/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json
@@ -8,7 +8,8 @@
"properties": {
"id" :{
"type" : "string",
- "description" : "Uniquely identifies each activity within the service"
+ "description" : "Uniquely identifies each activity within the service",
+ "required" : true
},
"actor" : {
"type": "object",
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json b/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json
index d51db27..eec09a8 100644
--- a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json
+++ b/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json
@@ -8,7 +8,7 @@
"id" : {
"type" : "string",
"description" : "Provides a permanent, universally unique identifier for the object in the form of an absolute IRI [RFC3987]. An object SHOULD contain a single id property. If an object does not contain an id property, consumers MAY use the value of the url property as a less-reliable, non-unique identifier.",
- "default" : "{link}"
+ "required" : true
},
"image" : {
"format":"image",
[28/38] incubator-streams git commit: STREAMS-204 | Added in a Google
Plus Deserializer and Processor for Comments. The processor takes any
comments found for the passed in Activity (via a to-be-seen provider) and
appends them as ActivityObjects. A total
Posted by sb...@apache.org.
STREAMS-204 | Added in a Google Plus Deserializer and Processor for Comments. The processor takes any comments found for the passed in Activity (via a to-be-seen provider) and appends them as ActivityObjects. A total "comment_count" attribute is also stored in the "extensions" section of the Activity.
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/08e21453
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/08e21453
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/08e21453
Branch: refs/heads/STREAMS-49
Commit: 08e21453084e79661f1687a47f1310ec44568eaa
Parents: 2358b4e
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Wed Oct 29 17:44:39 2014 -0500
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Thu Nov 6 14:04:27 2014 -0600
----------------------------------------------------------------------
.../processor/GooglePlusCommentProcessor.java | 88 +++++++++++++-
.../util/GPlusCommentDeserializer.java | 99 +++++++++++++++-
.../serializer/util/GooglePlusActivityUtil.java | 50 ++++++++
.../gplus/GooglePLusCommentSerDeTest.java | 115 ++++++++++++++++++-
.../resources/google_plus_comments_jsons.txt | 3 +
5 files changed, 343 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
index 7dc4d56..583c741 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
@@ -1,7 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus.processor;
-/**
- * Created by rdouglas on 10/29/14.
- */
-public class GooglePlusCommentProcessor {
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Comment;
+import com.google.api.services.plus.model.Person;
+import com.google.gplus.serializer.util.GPlusActivityDeserializer;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.pojo.json.Activity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class GooglePlusCommentProcessor implements StreamsProcessor {
+ private final static String STREAMS_ID = "GooglePlusCommentProcessor";
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentProcessor.class);
+ private GooglePlusActivityUtil googlePlusActivityUtil;
+ private int count;
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ StreamsDatum result = null;
+
+ try {
+ Object item = entry.getDocument();
+ LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
+
+ //Get G+ activity ID from our own activity ID
+ if (item instanceof Activity) {
+ Activity activity = (Activity) item;
+ String activityId = getGPlusID(activity.getId());
+
+ //Call Google Plus API to get list of comments for this activity ID
+ /**TODO: FILL ME OUT WITH THE API CALL**/
+ List<Comment> comments = Lists.newArrayList();
+
+ googlePlusActivityUtil.updateActivity(comments, activity);
+ result = new StreamsDatum(activity);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.error("Exception while converting Comment to Activity: {}", e.getMessage());
+ }
+
+ if( result != null )
+ return com.google.common.collect.Lists.newArrayList(result);
+ else
+ return com.google.common.collect.Lists.newArrayList();
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ googlePlusActivityUtil = new GooglePlusActivityUtil();
+ count = 0;
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ private String getGPlusID(String activityID) {
+ String[] activityParts = activityID.split(":");
+ return (activityParts.length > 0) ? activityParts[activityParts.length - 1] : "";
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
index 51446e0..513bdc7 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
@@ -1,7 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus.serializer.util;
-/**
- * Created by rdouglas on 10/29/14.
- */
-public class GPlusCommentDeserializer {
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.client.util.DateTime;
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Comment;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+public class GPlusCommentDeserializer extends JsonDeserializer<Comment> {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GPlusActivityDeserializer.class);
+
+ /**
+ * Because the GooglePlus Comment object {@link com.google.api.services.plus.model.Comment} contains complex objects
+ * within its hierarchy, we have to use a custom deserializer
+ *
+ * @param jsonParser
+ * @param deserializationContext
+ * @return The deserialized {@link com.google.api.services.plus.model.Comment} object
+ * @throws java.io.IOException
+ * @throws com.fasterxml.jackson.core.JsonProcessingException
+ */
+ @Override
+ public Comment deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+
+ JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+ ObjectMapper objectMapper = new StreamsJacksonMapper();
+ Comment comment = new Comment();
+
+ try {
+ comment.setEtag(node.get("etag").asText());
+ comment.setVerb(node.get("verb").asText());
+ comment.setId(node.get("id").asText());
+ comment.setPublished(DateTime.parseRfc3339(node.get("published").asText()));
+ comment.setUpdated(DateTime.parseRfc3339(node.get("updated").asText()));
+
+ Comment.Actor actor = new Comment.Actor();
+ JsonNode actorNode = node.get("actor");
+ actor.setDisplayName(actorNode.get("displayName").asText());
+ actor.setUrl(actorNode.get("url").asText());
+
+ Comment.Actor.Image image = new Comment.Actor.Image();
+ JsonNode imageNode = actorNode.get("image");
+ image.setUrl(imageNode.get("url").asText());
+
+ actor.setImage(image);
+
+ comment.setObject(objectMapper.readValue(objectMapper.writeValueAsString(node.get("object")), Comment.PlusObject.class));
+
+ comment.setSelfLink(node.get("selfLink").asText());
+
+ List<Comment.InReplyTo> replies = Lists.newArrayList();
+ for(JsonNode reply : node.get("inReplyTo")) {
+ Comment.InReplyTo r = objectMapper.readValue(objectMapper.writeValueAsString(reply), Comment.InReplyTo.class);
+ replies.add(r);
+ }
+
+ comment.setInReplyTo(replies);
+
+ Comment.Plusoners plusoners = new Comment.Plusoners();
+ JsonNode plusonersNode = node.get("plusoners");
+ plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
+ comment.setPlusoners(plusoners);
+ } catch (Exception e) {
+ LOGGER.error("Exception while trying to deserialize activity object: {}", e);
+ }
+
+ return comment;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
index 73b4b0d..4e330fa 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
@@ -19,6 +19,8 @@
package com.google.gplus.serializer.util;
+import com.google.api.client.util.Maps;
+import com.google.api.services.plus.model.Comment;
import com.google.api.services.plus.model.Person;
import org.apache.streams.pojo.json.*;
import org.apache.streams.pojo.json.Activity;
@@ -31,6 +33,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
+import java.util.List;
import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
@@ -58,6 +61,22 @@ public class GooglePlusActivityUtil {
}
/**
+ * Given a {@link List} of {@link com.google.api.services.plus.model.Comment} objects and an
+ * {@link org.apache.streams.pojo.json.Activity}, update that Activity to contain all comments
+ *
+ * @param comments
+ * @param activity
+ */
+ public static void updateActivity(List<Comment> comments, Activity activity) {
+ for(Comment comment : comments) {
+ addComment(activity, comment);
+ }
+
+ Map<String, Object> extensions = ensureExtensions(activity);
+ extensions.put("comment_count", comments.size());
+ }
+
+ /**
* Given a Google Plus {@link com.google.api.services.plus.model.Activity},
* convert that into an Activity streams formatted {@link org.apache.streams.pojo.json.Activity}
*
@@ -88,6 +107,37 @@ public class GooglePlusActivityUtil {
}
/**
+ * Adds a single {@link com.google.api.services.plus.model.Comment} to the Object.Attachments
+ * section of the passed in {@link org.apache.streams.pojo.json.Activity}
+ *
+ * @param activity
+ * @param comment
+ */
+ private static void addComment(Activity activity, Comment comment) {
+ ActivityObject obj = new ActivityObject();
+
+ obj.setId(comment.getId());
+ obj.setPublished(new DateTime(String.valueOf(comment.getPublished())));
+ obj.setUpdated(new DateTime(String.valueOf(comment.getUpdated())));
+ obj.setContent(comment.getObject().getContent());
+ obj.setObjectType(comment.getObject().getObjectType());
+
+ Map<String, Object> extensions = Maps.newHashMap();
+ extensions.put("googlePlus", comment);
+
+ obj.setAdditionalProperty("extensions", extensions);
+
+ if(activity.getObject() == null) {
+ activity.setObject(new ActivityObject());
+ }
+ if(activity.getObject().getAttachments() == null) {
+ activity.getObject().setAttachments(new ArrayList<ActivityObject>());
+ }
+
+ activity.getObject().getAttachments().add(obj);
+ }
+
+ /**
* Add in necessary extensions from the passed in {@link com.google.api.services.plus.model.Activity} to the
* {@link org.apache.streams.pojo.json.Activity} object
*
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
index b3479f9..9fea22c 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
@@ -1,7 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus;
-/**
- * Created by rdouglas on 10/29/14.
- */
-public class GooglePLusCommentSerDeTest {
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Comment;
+import com.google.gplus.serializer.util.GPlusCommentDeserializer;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class GooglePlusCommentSerDeTest {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentSerDeTest.class);
+ private ObjectMapper objectMapper;
+ private GooglePlusActivityUtil googlePlusActivityUtil;
+
+ @Before
+ public void setup() {
+ objectMapper = new StreamsJacksonMapper();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Comment.class, new GPlusCommentDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ googlePlusActivityUtil = new GooglePlusActivityUtil();
+ }
+
+ @org.junit.Test
+ public void testCommentObjects() {
+ InputStream is = GooglePlusPersonSerDeTest.class.getResourceAsStream("/google_plus_comments_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ Activity activity = new Activity();
+ List<Comment> comments = Lists.newArrayList();
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Comment comment = objectMapper.readValue(line, Comment.class);
+
+ LOGGER.info("comment: {}", comment);
+
+ assertNotNull(comment);
+ assertNotNull(comment.getEtag());
+ assertNotNull(comment.getId());
+ assertNotNull(comment.getInReplyTo());
+ assertNotNull(comment.getObject());
+ assertNotNull(comment.getPlusoners());
+ assertNotNull(comment.getPublished());
+ assertNotNull(comment.getUpdated());
+ assertNotNull(comment.getSelfLink());
+ assertEquals(comment.getVerb(), "post");
+
+ comments.add(comment);
+ }
+ }
+
+ assertEquals(comments.size(), 3);
+
+ googlePlusActivityUtil.updateActivity(comments, activity);
+ assertNotNull(activity);
+ assertNotNull(activity.getObject());
+ assertEquals(activity.getObject().getAttachments().size(), 3);
+ } catch (Exception e) {
+ LOGGER.error("Exception while testing serializability: {}", e);
+ }
+ }
+
+ @org.junit.Test
+ public void testEmptyComments() {
+ Activity activity = new Activity();
+
+ googlePlusActivityUtil.updateActivity(new ArrayList<Comment>(), activity);
+
+ assertNull(activity.getObject());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
index e69de29..e52abf1 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
@@ -0,0 +1,3 @@
+{ "kind": "plus#comment", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/ANtsF6kh7Ztc8-ufvOzWL234qfY\"", "verb": "post", "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k.1414517802192122", "published": "2014-10-28T17:36:42.192Z", "updated": "2014-10-28T17:36:42.192Z", "actor": { "id": "110922531601810522565", "displayName": "Adalberto Hernandez", "url": "https://plus.google.com/110922531601810522565", "image": { "url": "https://lh4.googleusercontent.com/-VpIxk0xr8i0/AAAAAAAAAAI/AAAAAAAADl4/HgEI76iIDPk/photo.jpg?sz=50" } }, "object": { "objectType": "comment", "content": "I would buy the nexus 6 I still think the screen size is perfect for me" }, "selfLink": "https://content.googleapis.com/plus/v1/comments/z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k#1414517802192122", "inReplyTo": [ { "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k", "url": "https://plus.google.com/101127444819996140229/posts/QXuZzrgguaK" } ], "plusoners": { "totalItems": 6 } }
+{ "kind": "plus#comment", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/81qALbxT1WteuRVngMKhNsvF2hM\"", "verb": "post", "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k.1414517816125433", "published": "2014-10-28T17:36:56.125Z", "updated": "2014-10-28T17:38:14.000Z", "actor": { "id": "107131626164728505473", "displayName": "Vance McAlister", "url": "https://plus.google.com/107131626164728505473", "image": { "url": "https://lh4.googleusercontent.com/-ENY-GPKzlpM/AAAAAAAAAAI/AAAAAAAAQPs/qTzDXHiDs6c/photo.jpg?sz=50" } }, "object": { "objectType": "comment", "content": "Only if they have gotten rid of Blur and it is as much pure Android as the Moto X." }, "selfLink": "https://content.googleapis.com/plus/v1/comments/z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k#1414517816125433", "inReplyTo": [ { "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k", "url": "https://plus.google.com/101127444819996140229/posts/QXuZzrgguaK" } ], "plusoners": { "totalItems": 7 } }, { "kind": "plus#comment", "etag": "\"Vea_b94Y77GDGgR
K7gFNPnolKQw/3tvcbp0tAYDvMPPViM-Iq8CsVLU\"", "verb": "post", "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k.1414517894284415", "published": "2014-10-28T17:38:14.284Z", "updated": "2014-10-28T17:38:14.284Z", "actor": { "id": "116526038399605835276", "displayName": "Thiago Vinhas", "url": "https://plus.google.com/116526038399605835276", "image": { "url": "https://lh6.googleusercontent.com/-ze6E8Zj8LM8/AAAAAAAAAAI/AAAAAAAAcII/2htWB7qbHs4/photo.jpg?sz=50" } }, "object": { "objectType": "comment", "content": "Verizon? No, thank you. I prefer to be kicked on the balls." }, "selfLink": "https://content.googleapis.com/plus/v1/comments/z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k#1414517894284415", "inReplyTo": [ { "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k", "url": "https://plus.google.com/101127444819996140229/posts/QXuZzrgguaK" } ], "plusoners": { "totalItems": 52 } }
+{ "kind": "plus#comment", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/DCSYhsbyQnRvdVQ8U7e1B6Rw9F8\"", "verb": "post", "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k.1414518523966061", "published": "2014-10-28T17:48:43.966Z", "updated": "2014-10-28T17:48:43.966Z", "actor": { "id": "115669911467014111133", "displayName": "Eduardo Castaneda", "url": "https://plus.google.com/115669911467014111133", "image": { "url": "https://lh4.googleusercontent.com/-I0ghHQLUDC4/AAAAAAAAAAI/AAAAAAAABFk/YkeCQCLzwk8/photo.jpg?sz=50" } }, "object": { "objectType": "comment", "content": "how the fuck is this not the new nexus?!?! " }, "selfLink": "https://content.googleapis.com/plus/v1/comments/z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k#1414518523966061", "inReplyTo": [ { "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k", "url": "https://plus.google.com/101127444819996140229/posts/QXuZzrgguaK" } ], "plusoners": { "totalItems": 18 } }
\ No newline at end of file
[16/38] incubator-streams git commit: STREAMS-202 | Additional
modifications
Posted by sb...@apache.org.
STREAMS-202 | Additional modifications
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/42d0ab33
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/42d0ab33
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/42d0ab33
Branch: refs/heads/STREAMS-49
Commit: 42d0ab33f20cada4bbf71eb590a4858a1f1cad2e
Parents: eba9aef
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Tue Oct 28 12:03:57 2014 -0500
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Tue Oct 28 12:03:57 2014 -0500
----------------------------------------------------------------------
.../google-gplus/pom.xml | 2 +-
.../processor/GooglePlusTypeConverter.java | 92 ++++++++++++++-
.../util/GPlusPersonDeserializer.java | 106 ++++++++++++++++-
.../serializer/util/GooglePlusActivityUtil.java | 116 ++++++++++++++++++-
.../google/gplus/GooglePlusPersonSerDeTest.java | 68 +++--------
.../processor/GooglePlusTypeConverterTest.java | 99 +++++++++++++++-
.../test/resources/google_plus_person_jsons.txt | 2 +
7 files changed, 414 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/42d0ab33/streams-contrib/streams-provider-google/google-gplus/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/pom.xml b/streams-contrib/streams-provider-google/google-gplus/pom.xml
index 2a236d9..0437408 100644
--- a/streams-contrib/streams-provider-google/google-gplus/pom.xml
+++ b/streams-contrib/streams-provider-google/google-gplus/pom.xml
@@ -67,7 +67,7 @@
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-plus</artifactId>
- <version>v1-rev118-1.17.0-rc</version>
+ <version>v1-rev184-1.19.0</version>
</dependency>
<dependency>
<groupId>com.google.api-client</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/42d0ab33/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
index f23ca17..928eec9 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
@@ -1,7 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus.processor;
-/**
- * Created by rdouglas on 10/27/14.
- */
-public class GooglePlusTypeConverter {
+import com.google.api.services.plus.model.Person;
+import com.google.common.collect.Lists;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.pojo.json.Activity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Queue;
+
+public class GooglePlusTypeConverter implements StreamsProcessor {
+ public final static String STREAMS_ID = "GooglePlusTypeConverter";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusTypeConverter.class);
+ private Queue<Person> inQueue;
+ private Queue<StreamsDatum> outQueue;
+ private GooglePlusActivityUtil googlePlusActivityUtil;
+ private int count = 0;
+
+ public GooglePlusTypeConverter() {}
+
+ public Queue<StreamsDatum> getProcessorOutputQueue() {
+ return outQueue;
+ }
+
+ public void setProcessorInputQueue(Queue<Person> inputQueue) {
+ inQueue = inputQueue;
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ StreamsDatum result = null;
+
+ try {
+ Object item = entry.getDocument();
+
+ LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
+ Activity activity = null;
+ if(item instanceof Person) {
+ activity = new Activity();
+
+ googlePlusActivityUtil.updateActivity((Person)item, activity);
+ }
+
+ if(activity != null) {
+ result = new StreamsDatum(activity);
+ count++;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.error("Exception while converting Person to Activity: {}", e.getMessage());
+ }
+
+ if( result != null )
+ return Lists.newArrayList(result);
+ else
+ return Lists.newArrayList();
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ googlePlusActivityUtil = new GooglePlusActivityUtil();
+ }
+
+ @Override
+ public void cleanUp() {
+ //No-op
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/42d0ab33/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
index 48fed99..7ee3990 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
@@ -1,7 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus.serializer.util;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.IntNode;
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Person;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
/**
- * Created by rdouglas on 10/28/14.
+ * Custom deserializer for GooglePlus' Person model
*/
-public class GPlusPersonDeserializer {
-}
+public class GPlusPersonDeserializer extends JsonDeserializer<Person> {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GPlusPersonDeserializer.class);
+
+ /**
+ * Because the GooglePlus Person object contains complex objects within its hierarchy, we have to use
+ * a custom deserializer
+ *
+ * @param jsonParser
+ * @param deserializationContext
+ * @return The deserialized {@link com.google.api.services.plus.model.Person} object
+ * @throws IOException
+ * @throws JsonProcessingException
+ */
+ @Override
+ public Person deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+ ObjectMapper m = new StreamsJacksonMapper();
+
+ JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+ Person person = new Person();
+ try {
+
+ person.setCircledByCount((Integer) ((IntNode) node.get("circledByCount")).numberValue());
+ person.setDisplayName(node.get("displayName").asText());
+ person.setEtag(node.get("etag").asText());
+ person.setGender(node.get("gender").asText());
+ person.setId(node.get("id").asText());
+
+ Person.Image image = new Person.Image();
+ JsonNode imageNode = node.get("image");
+ image.setIsDefault(imageNode.get("isDefault").asBoolean());
+ image.setUrl(imageNode.get("url").asText());
+ person.setImage(image);
+
+ person.setIsPlusUser(node.get("isPlusUser").asBoolean());
+ person.setKind(node.get("kind").asText());
+
+ JsonNode nameNode = node.get("name");
+ Person.Name name = m.readValue(m.writeValueAsString(nameNode), Person.Name.class);
+ person.setName(name);
+
+ person.setObjectType(node.get("objectType").asText());
+
+ List<Person.Organizations> organizations = Lists.newArrayList();
+ for (JsonNode orgNode : node.get("organizations")) {
+ Person.Organizations org = m.readValue(m.writeValueAsString(orgNode), Person.Organizations.class);
+ organizations.add(org);
+ }
+ person.setOrganizations(organizations);
+
+ person.setUrl(node.get("url").asText());
+ person.setVerified(node.get("verified").asBoolean());
+
+ List<Person.Emails> emails = Lists.newArrayList();
+ for (JsonNode emailNode : node.get("emails")) {
+ Person.Emails email = m.readValue(m.writeValueAsString(emailNode), Person.Emails.class);
+ emails.add(email);
+ }
+
+ person.setTagline(node.get("tagline").asText());
+ person.setAboutMe(node.get("aboutMe").asText());
+ } catch (Exception e) {
+ LOGGER.error("Exception while trying to deserialize a Person object: {}", e);
+ }
+
+ return person;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/42d0ab33/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
index a3150c8..3fce398 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
@@ -1,7 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus.serializer.util;
-/**
- * Created by rdouglas on 10/27/14.
- */
+import com.google.api.services.plus.model.Person;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.collect.Lists;
+import org.apache.streams.exceptions.ActivitySerializerException;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.apache.streams.pojo.json.Image;
+import org.apache.streams.pojo.json.Provider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
public class GooglePlusActivityUtil {
-}
+ private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusActivityUtil.class);
+
+ /**
+ * Given a Person object and an activity, fill out the appropriate actor details
+ *
+ * @param item
+ * @param activity
+ * @throws ActivitySerializerException
+ */
+ public static void updateActivity(Person item, Activity activity) throws ActivitySerializerException {
+ activity.setActor(buildActor(item));
+ activity.setVerb("update");
+
+ activity.setId(formatId(activity.getVerb(),
+ Optional.fromNullable(
+ item.getId())
+ .orNull()));
+
+ activity.setProvider(getProvider());
+ }
+
+ /**
+ * Extract the relevant details from the passed in Person object and build
+ * an actor with them
+ *
+ * @param person
+ * @return Actor constructed with relevant Person details
+ */
+ private static Actor buildActor(Person person) {
+ Actor actor = new Actor();
+
+ actor.setUrl(person.getUrl());
+ actor.setDisplayName(person.getDisplayName());
+ actor.setId(formatId(String.valueOf(person.getId())));
+
+ if(person.getAboutMe() != null) {
+ actor.setSummary(person.getAboutMe());
+ } else if(person.getTagline() != null) {
+ actor.setSummary(person.getTagline());
+ }
+
+ Image image = new Image();
+ Person.Image googlePlusImage = person.getImage();
+
+ if(googlePlusImage != null) {
+ image.setUrl(googlePlusImage.getUrl());
+ }
+ actor.setImage(image);
+
+ Map<String, Object> extensions = new HashMap<String, Object>();
+
+ extensions.put("followers", person.getCircledByCount());
+ extensions.put("googleplus", person);
+ actor.setAdditionalProperty("extensions", extensions);
+
+ return actor;
+ }
+
+ /**
+ * Gets the common googleplus {@link org.apache.streams.pojo.json.Provider} object
+ * @return a provider object representing GooglePlus
+ */
+ public static Provider getProvider() {
+ Provider provider = new Provider();
+ provider.setId("id:providers:googleplus");
+ provider.setDisplayName("GooglePlus");
+ return provider;
+ }
+
+ /**
+ * Formats the ID to conform with the Apache Streams activity ID convention
+ * @param idparts the parts of the ID to join
+ * @return a valid Activity ID in format "id:googleplus:part1:part2:...partN"
+ */
+ public static String formatId(String... idparts) {
+ return Joiner.on(":").join(Lists.asList("id:googleplus", idparts));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/42d0ab33/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeTest.java
index 22cc340..6f6b2e3 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeTest.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeTest.java
@@ -19,54 +19,55 @@
package com.google.gplus;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.fasterxml.jackson.databind.node.IntNode;
-import com.google.api.client.util.Lists;
import com.google.api.services.plus.model.Person;
+import com.google.gplus.serializer.util.GPlusPersonDeserializer;
import com.google.gplus.serializer.util.GooglePlusActivityUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.Activity;
import org.apache.streams.pojo.json.Actor;
import org.apache.streams.pojo.json.Provider;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
-import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
public class GooglePlusPersonSerDeTest {
private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusPersonSerDeTest.class);
+ private ObjectMapper objectMapper;
+ private GooglePlusActivityUtil googlePlusActivityUtil;
- @Test
- public void TestPersonObjects() {
- ObjectMapper objectMapper = new StreamsJacksonMapper();
+ @Before
+ public void setup() {
+ objectMapper = new StreamsJacksonMapper();
SimpleModule simpleModule = new SimpleModule();
simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ googlePlusActivityUtil = new GooglePlusActivityUtil();
+ }
+
+ @Test
+ public void TestPersonObjects() {
InputStream is = GooglePlusPersonSerDeTest.class.getResourceAsStream("/google_plus_person_jsons.txt");
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
- GooglePlusActivityUtil googlePlusActivityUtil = new GooglePlusActivityUtil();
-
try {
while (br.ready()) {
String line = br.readLine();
if (!StringUtils.isEmpty(line)) {
LOGGER.info("raw: {}", line);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
Activity activity = new Activity();
Person person = objectMapper.readValue(line, Person.class);
@@ -93,47 +94,4 @@ public class GooglePlusPersonSerDeTest {
LOGGER.error("Exception while testing serializability: {}", e);
}
}
-}
-
-class GPlusPersonDeserializer extends JsonDeserializer<Person> {
- @Override
- public Person deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
- ObjectMapper m = new StreamsJacksonMapper();
-
- JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- Person person = new Person();
-
- person.setCircledByCount((Integer)((IntNode)node.get("circledByCount")).numberValue());
- person.setDisplayName(node.get("displayName").asText());
- person.setEtag(node.get("etag").asText());
- person.setGender(node.get("gender").asText());
- person.setId(node.get("id").asText());
-
- Person.Image image = new Person.Image();
- JsonNode imageNode = node.get("image");
- image.setIsDefault(imageNode.get("isDefault").asBoolean());
- image.setUrl(imageNode.get("url").asText());
- person.setImage(image);
-
- person.setIsPlusUser(node.get("isPlusUser").asBoolean());
- person.setKind(node.get("kind").asText());
-
- JsonNode nameNode = node.get("name");
- Person.Name name = m.readValue(m.writeValueAsString(nameNode), Person.Name.class);
- person.setName(name);
-
- person.setObjectType(node.get("objectType").asText());
-
- List<Person.Organizations> organizations = Lists.newArrayList();
- for(JsonNode orgNode : node.get("organizations")) {
- Person.Organizations org = m.readValue(m.writeValueAsString(orgNode), Person.Organizations.class);
- organizations.add(org);
- }
- person.setOrganizations(organizations);
-
- person.setUrl(node.get("url").asText());
- person.setVerified(node.get("verified").asBoolean());
-
- return person;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/42d0ab33/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
index e4923ae..6fbdf19 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
@@ -1,7 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus.processor;
-/**
- * Created by rdouglas on 10/28/14.
- */
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.plus.model.Person;
+import com.google.gplus.serializer.util.GPlusPersonDeserializer;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
public class GooglePlusTypeConverterTest {
-}
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusTypeConverterTest.class);
+ private GooglePlusTypeConverter googlePlusTypeConverter;
+ private ObjectMapper objectMapper;
+
+ @Before
+ public void setup() {
+ objectMapper = new StreamsJacksonMapper();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ googlePlusTypeConverter = new GooglePlusTypeConverter();
+ googlePlusTypeConverter.prepare(null);
+ }
+
+ @Test
+ public void testProcess() {
+ InputStream is = GooglePlusTypeConverterTest.class.getResourceAsStream("/google_plus_person_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Activity activity = new Activity();
+
+ Person person = objectMapper.readValue(line, Person.class);
+ StreamsDatum streamsDatum = new StreamsDatum(person);
+
+ assertNotNull(streamsDatum.getDocument());
+
+ List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
+ GooglePlusActivityUtil.updateActivity(person, activity);
+
+ assertEquals(retList.size(), 1);
+ assert(retList.get(0).getDocument() instanceof Activity);
+ assertEquals(activity, retList.get(0).getDocument());
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.error("Exception testing the GooglePlusTypeConverter: {}", e);
+ }
+ }
+
+ @Test
+ public void testEmptyProcess() {
+ List<StreamsDatum> retList = googlePlusTypeConverter.process(null);
+
+ assertEquals(retList.size(), 0);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/42d0ab33/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_person_jsons.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_person_jsons.txt b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_person_jsons.txt
index e69de29..3280747 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_person_jsons.txt
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_person_jsons.txt
@@ -0,0 +1,2 @@
+{"circledByCount":69,"displayName":"Robert Douglas","etag":"\"Vea_b94Y77GDGgRK7gFNPnolKQw/H7iVXOYgK9fQa4LWUIeauVGCMNE\"","gender":"male","id":"106406455118529211676","image":{"isDefault":false,"url":"https://lh3.googleusercontent.com/-u-IN7x710H8/AAAAAAAAAAI/AAAAAAAAAQE/tdo5WzJuBiM/photo.jpg?sz=50"},"isPlusUser":true,"kind":"plus#person","name":{"familyName":"Douglas","givenName":"Robert"},"objectType":"person","organizations":[{"name":"Johns Hopkins University","primary":true,"title":"Computer Science","type":"school"}],"url":"https://plus.google.com/106406455118529211676","verified":false}
+{ "kind": "plus#person", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/_2kdKCZGyG8t19G4WeQrtqVZrHU\"", "gender": "male", "emails": [ { "value": "steve@blackmon.org", "type": "account" } ], "urls": [ { "value": "http://www.facebook.com/steveblackmon", "type": "otherProfile", "label": "steveblackmon" }, { "value": "http://twitter.com/steveblackmon", "type": "otherProfile", "label": "steveblackmon" }, { "value": "http://www.linkedin.com/in/blackmon", "type": "otherProfile", "label": "blackmon" }, { "value": "http://streams.incubator.apache.org/", "type": "contributor", "label": "Apache Streams" } ], "objectType": "person", "id": "118376908737486995861", "displayName": "Steve Blackmon", "name": { "familyName": "Blackmon", "givenName": "Steve" }, "tagline": "I am a talented hardware/software/systems engineer with a fervent interest in emerging technologies, and the economics of every imaginable industry sector.", "aboutMe": "<span>I enjoy thinking, reading, writing, coding, and communicating. M
y passions are science, technology, business, their mutual coevolution, and their collective impact on society and economic development. I love traveling, meeting new people, being outdoors, and most cultural activities.<br /></span>", "url": "https://plus.google.com/+SteveBlackmon", "image": { "url": "https://lh3.googleusercontent.com/-XdUIqdMkCWA/AAAAAAAAAAI/AAAAAAAAAAA/4252rscbv5M/photo.jpg?sz=50", "isDefault": true }, "organizations": [ { "name": "McCombs School of Business", "title": "MBA", "type": "school", "endDate": "2013", "primary": false }, { "name": "University of Southern California", "title": "MS Computer Science", "type": "school", "endDate": "2005", "primary": false }, { "name": "University of Texas at Austin", "title": "BS Computer Engineering", "type": "school", "endDate": "2003", "primary": false }, { "name": "W2O Group", "title": "Director", "type": "work", "primary": true } ], "placesLived": [ { "value": "Austin, TX", "primary": true } ], "isPlusUser": true, "la
nguage": "en", "circledByCount": 139, "verified": false, "domain": "blackmon.org" }
\ No newline at end of file
[07/38] incubator-streams git commit: introduced a
SimpleHTTPGetProvider reorganized configuration added access_token support
added http basic auth support
Posted by sb...@apache.org.
introduced a SimpleHTTPGetProvider
reorganized configuration
added access_token support
added http basic auth support
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d62061de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d62061de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d62061de
Branch: refs/heads/STREAMS-49
Commit: d62061dec8628484e66d3494e02f1f65efafda41
Parents: a576460
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Oct 12 17:49:01 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Oct 12 17:49:01 2014 -0500
----------------------------------------------------------------------
streams-components/pom.xml | 2 +-
streams-components/streams-http/README.md | 16 ++
streams-components/streams-http/pom.xml | 159 +++++++++++++
.../components/http/HttpConfigurator.java | 62 +++++
.../http/processor/SimpleHTTPGetProcessor.java | 230 +++++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 215 +++++++++++++++++
.../components/http/HttpConfiguration.json | 50 ++++
.../http/HttpProcessorConfiguration.json | 28 +++
.../http/HttpProviderConfiguration.json | 18 ++
.../streams-processor-http/README.md | 16 --
.../streams-processor-http/pom.xml | 154 -------------
.../components/http/HttpConfigurator.java | 53 -----
.../components/http/SimpleHTTPGetProcessor.java | 218 ------------------
.../HttpProcessorConfiguration.json | 47 ----
14 files changed, 779 insertions(+), 489 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/pom.xml b/streams-components/pom.xml
index 26384b1..9942e14 100644
--- a/streams-components/pom.xml
+++ b/streams-components/pom.xml
@@ -37,7 +37,7 @@
</properties>
<modules>
- <module>streams-processor-http</module>
+ <module>streams-http</module>
</modules>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/README.md
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/README.md b/streams-components/streams-http/README.md
new file mode 100644
index 0000000..62dd4c1
--- /dev/null
+++ b/streams-components/streams-http/README.md
@@ -0,0 +1,16 @@
+streams-processor-http
+=====================
+
+Hit an http endpoint and place the result in extensions
+
+Example SimpleHTTPGetProcessor configuration:
+
+ "http": {
+ "protocol": "http",
+ "hostname": "urls.api.twitter.com",
+ "port": 9300,
+ "resourceUri": "1/urls/count.json"
+ }
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/pom.xml b/streams-components/streams-http/pom.xml
new file mode 100644
index 0000000..9c2b079
--- /dev/null
+++ b/streams-components/streams-http/pom.xml
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-components</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>streams-http</artifactId>
+
+ <name>streams-http</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-core</artifactId>
+ <type>jar</type>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.validation</groupId>
+ <artifactId>validation-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo-extensions</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.3.5</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-source-jaxb2</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jaxb2</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>org.apache.streams.http</targetPackage>
+ <useLongIntegers>true</useLongIntegers>
+ <useJodaDates>true</useJodaDates>
+ <includeJsr303Annotations>true</includeJsr303Annotations>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
new file mode 100644
index 0000000..900831f
--- /dev/null
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.components.http;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigRenderOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
+ */
+public class HttpConfigurator {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(HttpConfigurator.class);
+
+ private final static ObjectMapper mapper = new ObjectMapper();
+
+ public static HttpProviderConfiguration detectProviderConfiguration(Config config) {
+
+ HttpProviderConfiguration httpProviderConfiguration = null;
+
+ try {
+ httpProviderConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), HttpProviderConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse http configuration", e.getMessage());
+ }
+ return httpProviderConfiguration;
+ }
+
+ public static HttpProcessorConfiguration detectProcessorConfiguration(Config config) {
+
+ HttpProcessorConfiguration httpProcessorConfiguration = null;
+
+ try {
+ httpProcessorConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), HttpProcessorConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse http configuration", e.getMessage());
+ }
+ return httpProcessorConfiguration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
new file mode 100644
index 0000000..c2bfef6
--- /dev/null
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
@@ -0,0 +1,230 @@
+package org.apache.streams.components.http.processor;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.streams.components.http.HttpConfigurator;
+import org.apache.streams.components.http.HttpProcessorConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.data.util.ExtensionUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.Validation;
+import javax.validation.ValidatorFactory;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Processor retrieves contents from an known url and stores the resulting object in an extension field
+ */
+public class SimpleHTTPGetProcessor implements StreamsProcessor {
+
+ private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
+
+ // from root config id
+ private final static String EXTENSION = "account_type";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPGetProcessor.class);
+
+ protected ObjectMapper mapper;
+
+ protected URIBuilder uriBuilder;
+
+ protected CloseableHttpClient httpclient;
+
+ protected HttpProcessorConfiguration configuration;
+
+ protected String authHeader;
+//
+// // authorized only
+// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
+// //private String authHeader;
+//
+ public SimpleHTTPGetProcessor() {
+ this(HttpConfigurator.detectProcessorConfiguration(StreamsConfigurator.config.getConfig("http")));
+ }
+
+ public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
+ LOGGER.info("creating SimpleHTTPGetProcessor");
+ LOGGER.info(processorConfiguration.toString());
+ this.configuration = processorConfiguration;
+ }
+
+
+ /**
+ Override this to store a result other than exact json representation of response
+ */
+ protected ObjectNode prepareExtensionFragment(String entityString) {
+
+ try {
+ return mapper.readValue(entityString, ObjectNode.class);
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ }
+ }
+
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode getRootDocument(StreamsDatum datum) {
+
+ try {
+ String json = datum.getDocument() instanceof String ?
+ (String) datum.getDocument() :
+ mapper.writeValueAsString(datum.getDocument());
+ return mapper.readValue(json, ObjectNode.class);
+ } catch (JsonProcessingException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ }
+
+ }
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode getEntityToExtend(ObjectNode rootDocument) {
+
+ if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
+ return rootDocument;
+ else
+ return (ObjectNode) rootDocument.get(this.configuration.getEntity().toString());
+
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+
+ List<StreamsDatum> result = Lists.newArrayList();
+
+ ObjectNode rootDocument = getRootDocument(entry);
+
+ Map<String, String> params = prepareParams(entry);
+
+ URI uri;
+ for( Map.Entry<String,String> param : params.entrySet()) {
+ uriBuilder = uriBuilder.setParameter(param.getKey(), param.getValue());
+ }
+ try {
+ uri = uriBuilder.build();
+ } catch (URISyntaxException e) {
+ LOGGER.error("URI error {}", uriBuilder.toString());
+ return result;
+ }
+
+ HttpGet httpget = prepareHttpGet(uri);
+
+ CloseableHttpResponse response = null;
+
+ String entityString = null;
+ try {
+ response = httpclient.execute(httpget);
+ HttpEntity entity = response.getEntity();
+ // TODO: handle retry
+ if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
+ entityString = EntityUtils.toString(entity);
+ }
+ } catch (IOException e) {
+ LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, e.getMessage());
+ return result;
+ } finally {
+ try {
+ response.close();
+ } catch (IOException e) {}
+ try {
+ httpclient.close();
+ } catch (IOException e) {}
+ }
+
+ if( entityString == null )
+ return result;
+
+ LOGGER.debug(entityString);
+
+ ObjectNode extensionFragment = prepareExtensionFragment(entityString);
+
+ ObjectNode extensionEntity = getEntityToExtend(rootDocument);
+
+ ExtensionUtil.ensureExtensions(extensionEntity);
+
+ ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
+
+ entry.setDocument(rootDocument);
+
+ result.add(entry);
+
+ return result;
+
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+
+ return Maps.newHashMap();
+ }
+
+
+ public HttpGet prepareHttpGet(URI uri) {
+ HttpGet httpget = new HttpGet(uri);
+ httpget.addHeader("content-type", this.configuration.getContentType());
+ if( !Strings.isNullOrEmpty(authHeader))
+ httpget.addHeader("Authorization", String.format("Basic %s", authHeader));
+ return httpget;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
+ Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
+
+ mapper = StreamsJacksonMapper.getInstance();
+
+ uriBuilder = new URIBuilder()
+ .setScheme(this.configuration.getProtocol())
+ .setHost(this.configuration.getHostname())
+ .setPath(this.configuration.getResourcePath());
+
+ if( !Strings.isNullOrEmpty(configuration.getAccessToken()) )
+ uriBuilder = uriBuilder.addParameter("access_token", configuration.getAccessToken());
+ if( !Strings.isNullOrEmpty(configuration.getUsername())
+ && !Strings.isNullOrEmpty(configuration.getPassword())) {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append(configuration.getUsername());
+ stringBuilder.append(":");
+ stringBuilder.append(configuration.getPassword());
+ String string = stringBuilder.toString();
+ authHeader = Base64.encodeBase64String(string.getBytes());
+ }
+ httpclient = HttpClients.createDefault();
+ }
+
+ @Override
+ public void cleanUp() {
+ LOGGER.info("shutting down SimpleHTTPGetProcessor");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
new file mode 100644
index 0000000..622225a
--- /dev/null
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
@@ -0,0 +1,215 @@
+package org.apache.streams.components.http.provider;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.streams.components.http.HttpConfigurator;
+import org.apache.streams.components.http.HttpProviderConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Provider retrieves contents from an known set of urls and passes all resulting objects downstream
+ */
+public class SimpleHTTPGetProvider implements StreamsProvider {
+
+ private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
+
+ // from root config id
+ private final static String EXTENSION = "account_type";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPGetProvider.class);
+
+ protected ObjectMapper mapper;
+
+ protected URIBuilder uriBuilder;
+
+ protected CloseableHttpClient httpclient;
+
+ protected HttpProviderConfiguration configuration;
+
+ protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
+
+ // // authorized only
+// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
+// //private String authHeader;
+//
+ public SimpleHTTPGetProvider() {
+ this(HttpConfigurator.detectProviderConfiguration(StreamsConfigurator.config.getConfig("http")));
+ }
+
+ public SimpleHTTPGetProvider(HttpProviderConfiguration providerConfiguration) {
+ LOGGER.info("creating SimpleHTTPGetProvider");
+ LOGGER.info(providerConfiguration.toString());
+ this.configuration = providerConfiguration;
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+
+ return Maps.newHashMap();
+ }
+
+ public HttpGet prepareHttpGet(URI uri) {
+ HttpGet httpget = new HttpGet(uri);
+ httpget.addHeader("content-type", this.configuration.getContentType());
+ return httpget;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+// ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
+// Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
+
+ mapper = StreamsJacksonMapper.getInstance();
+
+ uriBuilder = new URIBuilder()
+ .setScheme(this.configuration.getProtocol())
+ .setHost(this.configuration.getHostname())
+ .setPath(this.configuration.getResourcePath());
+
+ httpclient = HttpClients.createDefault();
+ }
+
+ @Override
+ public void cleanUp() {
+
+ LOGGER.info("shutting down SimpleHTTPGetProvider");
+ try {
+ httpclient.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ httpclient.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ httpclient = null;
+ }
+ }
+ }
+
+ @Override
+ public void startStream() {
+
+ }
+
+ @Override
+ public StreamsResultSet readCurrent() {
+ StreamsResultSet current;
+
+ uriBuilder = uriBuilder.setPath(
+ Joiner.on("/").skipNulls().join(uriBuilder.getPath(), configuration.getResource(), configuration.getResourcePostfix())
+ );
+
+ URI uri;
+ try {
+ uri = uriBuilder.build();
+ } catch (URISyntaxException e) {
+ uri = null;
+ }
+
+ List<ObjectNode> results = executeGet(uri);
+
+ lock.writeLock().lock();
+
+ for( ObjectNode item : results ) {
+ providerQueue.add(new StreamsDatum(item, item.get("id").asText(), new DateTime(item.get("timestamp").asText())));
+ }
+
+ LOGGER.debug("Creating new result set for {} items", providerQueue.size());
+ current = new StreamsResultSet(providerQueue);
+
+ return current;
+ }
+
+ protected List<ObjectNode> executeGet(URI uri) {
+
+ Preconditions.checkNotNull(uri);
+
+ List<ObjectNode> results = new ArrayList<>();
+
+ HttpGet httpget = prepareHttpGet(uri);
+
+ CloseableHttpResponse response = null;
+
+ String entityString = null;
+ try {
+ response = httpclient.execute(httpget);
+ HttpEntity entity = response.getEntity();
+ // TODO: handle retry
+ if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
+ entityString = EntityUtils.toString(entity);
+ if( !entityString.equals("{}") && !entityString.equals("[]") ) {
+ JsonNode jsonNode = mapper.readValue(entityString, JsonNode.class);
+ if (jsonNode != null && jsonNode instanceof ObjectNode ) {
+
+ results.add((ObjectNode) jsonNode);
+ } else if (jsonNode != null && jsonNode instanceof ArrayNode) {
+ ArrayNode arrayNode = (ArrayNode) jsonNode;
+ Iterator<JsonNode> iterator = arrayNode.elements();
+ while (iterator.hasNext()) {
+ ObjectNode element = (ObjectNode) iterator.next();
+
+ results.add(element);
+ }
+ }
+ }
+ }
+ } catch (IOException e) {
+ LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, e.getMessage());
+ } finally {
+ try {
+ response.close();
+ } catch (IOException e) {}
+ }
+ return results;
+ }
+
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ return null;
+ }
+
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ return null;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json
new file mode 100644
index 0000000..b4dc243
--- /dev/null
+++ b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json
@@ -0,0 +1,50 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.components.http.HttpConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "protocol": {
+ "type": "string",
+ "description": "Protocol",
+ "default": "http"
+ },
+ "hostname": {
+ "type": "string",
+ "description": "Hostname",
+ "required" : true
+ },
+ "port": {
+ "type": "integer",
+ "description": "Port",
+ "default": 80
+ },
+ "resourcePath": {
+ "type": "string",
+ "description": "Resource Path",
+ "required" : true
+ },
+ "content-type": {
+ "type": "string",
+ "description": "Resource content-type",
+ "required" : true,
+ "default": "application/json"
+ },
+ "access_token": {
+ "type": "string",
+ "description": "Known Access Token",
+ "required" : false
+ },
+ "username": {
+ "type": "string",
+ "description": "Basic Auth Username",
+ "required" : false
+ },
+ "password": {
+ "type": "string",
+ "description": "Basic Auth Password",
+ "required" : false
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json
new file mode 100644
index 0000000..32e4c23
--- /dev/null
+++ b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json
@@ -0,0 +1,28 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.components.http.HttpProcessorConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "extends": { "$ref": "HttpConfiguration.json" },
+ "properties": {
+ "entity": {
+ "type": "string",
+ "description": "Entity to extend",
+ "enum": [ "activity", "actor", "object", "target" ],
+ "required" : true,
+ "default": "activity"
+ },
+ "extension": {
+ "type": "string",
+ "description": "Extension identifier",
+ "required" : true
+ },
+ "urlField": {
+ "type": "string",
+ "description": "Field where url is located",
+ "required" : true,
+ "default": "url"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json
new file mode 100644
index 0000000..2c135d9
--- /dev/null
+++ b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json
@@ -0,0 +1,18 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.components.http.HttpProviderConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "extends": { "$ref": "HttpConfiguration.json" },
+ "properties": {
+ "resource": {
+ "type": "string",
+ "required" : false
+ },
+ "resourcePostfix": {
+ "type": "string",
+ "required" : false
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/README.md
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/README.md b/streams-components/streams-processor-http/README.md
deleted file mode 100644
index 62dd4c1..0000000
--- a/streams-components/streams-processor-http/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-streams-processor-http
-=====================
-
-Hit an http endpoint and place the result in extensions
-
-Example SimpleHTTPGetProcessor configuration:
-
- "http": {
- "protocol": "http",
- "hostname": "urls.api.twitter.com",
- "port": 9300,
- "resourceUri": "1/urls/count.json"
- }
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/pom.xml b/streams-components/streams-processor-http/pom.xml
deleted file mode 100644
index d9215ad..0000000
--- a/streams-components/streams-processor-http/pom.xml
+++ /dev/null
@@ -1,154 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-components</artifactId>
- <version>0.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>streams-processor-http</artifactId>
-
- <name>streams-processor-http</name>
-
- <dependencies>
-
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.3.5</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.8</version>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jsonschema2pojo</source>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>add-source-jaxb2</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jaxb2</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <configuration>
- <addCompileSourceRoot>true</addCompileSourceRoot>
- <generateBuilders>true</generateBuilders>
- <sourcePaths>
- <sourcePath>src/main/jsonschema</sourcePath>
- </sourcePaths>
- <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>org.apache.streams.http</targetPackage>
- <useLongIntegers>true</useLongIntegers>
- <useJodaDates>true</useJodaDates>
- <includeJsr303Annotations>true</includeJsr303Annotations>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
deleted file mode 100644
index 36801b8..0000000
--- a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.components.http;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
- */
-public class HttpConfigurator {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(HttpConfigurator.class);
-
- private final static ObjectMapper mapper = new ObjectMapper();
-
- public static HttpProcessorConfiguration detectConfiguration(Config config) {
-
- HttpProcessorConfiguration httpProcessorConfiguration = null;
-
- try {
- httpProcessorConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), HttpProcessorConfiguration.class);
- } catch (Exception e) {
- e.printStackTrace();
- LOGGER.warn("Could not parse http configuration", e.getMessage());
- }
- return httpProcessorConfiguration;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
deleted file mode 100644
index d74793a..0000000
--- a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
+++ /dev/null
@@ -1,218 +0,0 @@
-package org.apache.streams.components.http;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigValue;
-import org.apache.commons.io.IOUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.data.util.ActivityUtil;
-import org.apache.streams.data.util.ExtensionUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.validation.Validation;
-import javax.validation.ValidatorFactory;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-public class SimpleHTTPGetProcessor implements StreamsProcessor {
-
- private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
-
- // from root config id
- private final static String EXTENSION = "account_type";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPGetProcessor.class);
-
- protected ObjectMapper mapper;
-
- protected URIBuilder uriBuilder;
-
- protected CloseableHttpClient httpclient;
-
- protected HttpProcessorConfiguration configuration;
-//
-// // authorized only
-// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
-// //private String authHeader;
-//
- public SimpleHTTPGetProcessor() {
- LOGGER.info("creating SimpleHTTPGetProcessor");
- this.configuration = HttpConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("http"));
- }
-
- public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
- LOGGER.info("creating SimpleHTTPGetProcessor");
- LOGGER.info(processorConfiguration.toString());
- this.configuration = processorConfiguration;
- }
-
- /**
- Override this to add parameters to the request
- */
- protected Map<String, String> prepareParams(StreamsDatum entry) {
-
- return Maps.newHashMap();
- }
-
- /**
- Override this to store a result other than exact json representation of response
- */
- protected ObjectNode prepareExtensionFragment(String entityString) {
-
- try {
- return mapper.readValue(entityString, ObjectNode.class);
- } catch (IOException e) {
- LOGGER.warn(e.getMessage());
- return null;
- }
- }
-
- /**
- Override this to place result in non-standard location on document
- */
- protected ObjectNode getRootDocument(StreamsDatum datum) {
-
- try {
- String json = datum.getDocument() instanceof String ?
- (String) datum.getDocument() :
- mapper.writeValueAsString(datum.getDocument());
- return mapper.readValue(json, ObjectNode.class);
- } catch (JsonProcessingException e) {
- LOGGER.warn(e.getMessage());
- return null;
- } catch (IOException e) {
- LOGGER.warn(e.getMessage());
- return null;
- }
-
- }
- /**
- Override this to place result in non-standard location on document
- */
- protected ObjectNode getEntityToExtend(ObjectNode rootDocument) {
-
- if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
- return rootDocument;
- else
- return (ObjectNode) rootDocument.get(this.configuration.getEntity().toString());
-
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = Lists.newArrayList();
-
- ObjectNode rootDocument = getRootDocument(entry);
-
- Map<String, String> params = prepareParams(entry);
-
- URI uri;
- for( Map.Entry<String,String> param : params.entrySet()) {
- uriBuilder = uriBuilder.setParameter(param.getKey(), param.getValue());
- }
- try {
- uri = uriBuilder.build();
- } catch (URISyntaxException e) {
- LOGGER.error("URI error {}", uriBuilder.toString());
- return result;
- }
-
- HttpGet httpget = prepareHttpGet(uri);
-
- CloseableHttpResponse response = null;
-
- String entityString = null;
- try {
- response = httpclient.execute(httpget);
- HttpEntity entity = response.getEntity();
- // TODO: handle retry
- if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
- entityString = EntityUtils.toString(entity);
- }
- } catch (IOException e) {
- LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, e.getMessage());
- return result;
- } finally {
- try {
- response.close();
- } catch (IOException e) {}
- try {
- httpclient.close();
- } catch (IOException e) {}
- }
-
- if( entityString == null )
- return result;
-
- LOGGER.debug(entityString);
-
- ObjectNode extensionFragment = prepareExtensionFragment(entityString);
-
- ObjectNode extensionEntity = getEntityToExtend(rootDocument);
-
- ExtensionUtil.ensureExtensions(extensionEntity);
-
- ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
-
- entry.setDocument(rootDocument);
-
- result.add(entry);
-
- return result;
-
- }
-
- public HttpGet prepareHttpGet(URI uri) {
- HttpGet httpget = new HttpGet(uri);
- httpget.addHeader("content-type", this.configuration.getContentType());
- return httpget;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
- Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
-
- mapper = StreamsJacksonMapper.getInstance();
-
- uriBuilder = new URIBuilder()
- .setScheme(this.configuration.getProtocol())
- .setHost(this.configuration.getHostname())
- .setPath(this.configuration.getResourceUri());
-
- httpclient = HttpClients.createDefault();
- }
-
- @Override
- public void cleanUp() {
- LOGGER.info("shutting down SimpleHTTPGetProcessor");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json b/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
deleted file mode 100644
index 40c3bcd..0000000
--- a/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "javaType" : "org.apache.streams.components.http.HttpProcessorConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "Protocol",
- "default": "http"
- },
- "hostname": {
- "type": "string",
- "description": "Hostname",
- "required" : true
- },
- "port": {
- "type": "integer",
- "description": "Port",
- "default": 80
- },
- "resourceUri": {
- "type": "string",
- "description": "Resource URI",
- "required" : true
- },
- "content-type": {
- "type": "string",
- "description": "Resource URI",
- "required" : true,
- "default": "application/json"
- },
- "entity": {
- "type": "string",
- "description": "Entity to extend",
- "enum": [ "activity", "actor", "object", "target" ],
- "required" : true,
- "default": "activity"
- },
- "extension": {
- "type": "string",
- "description": "Extension identifier",
- "required" : true
- }
- }
-}
\ No newline at end of file
[04/38] incubator-streams git commit: class need not be abstract
rootDocument is what should exit processor
Posted by sb...@apache.org.
class need not be abstract
rootDocument is what should exit processor
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/a5764609
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/a5764609
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/a5764609
Branch: refs/heads/STREAMS-49
Commit: a576460911464cf260e37886ce840f4e57dd5f30
Parents: b8ccf9f
Author: sblackmon <sb...@apache.org>
Authored: Tue Sep 16 14:24:53 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Sep 16 14:24:53 2014 -0500
----------------------------------------------------------------------
.../apache/streams/components/http/SimpleHTTPGetProcessor.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/a5764609/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
index dec9d03..d74793a 100644
--- a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
@@ -40,7 +40,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
+public class SimpleHTTPGetProcessor implements StreamsProcessor {
private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
@@ -152,7 +152,7 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
try {
response = httpclient.execute(httpget);
HttpEntity entity = response.getEntity();
- // TODO: handle rate-limiting
+ // TODO: handle retry
if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
entityString = EntityUtils.toString(entity);
}
@@ -181,7 +181,7 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
- entry.setDocument(extensionEntity);
+ entry.setDocument(rootDocument);
result.add(entry);
[17/38] incubator-streams git commit: STREAMS-203 | Created
mechanisms to convert a GooglePlus Activity object to a Streams Activity
object. Updated serializers and TypeConverters accordingly. Added in
appropriate unit tests. Updated documentation.
Posted by sb...@apache.org.
STREAMS-203 | Created mechanisms to convert a GooglePlus Activity object to a Streams Activity object. Updated serializers and TypeConverters accordingly. Added in appropriate unit tests. Updated documentation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/c921f674
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/c921f674
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/c921f674
Branch: refs/heads/STREAMS-49
Commit: c921f6741804f745e6d4926166bf1cf792603bbd
Parents: 42d0ab3
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Tue Oct 28 17:42:39 2014 -0500
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Tue Oct 28 17:42:39 2014 -0500
----------------------------------------------------------------------
.../processor/GooglePlusTypeConverter.java | 5 +-
.../gplus/provider/GPlusActivitySerializer.java | 77 +---------
.../gplus/provider/GPlusEventProcessor.java | 3 -
.../util/GPlusActivityDeserializer.java | 129 +++++++++++++++++
.../serializer/util/GooglePlusActivityUtil.java | 143 +++++++++++++++++--
.../processor/GooglePlusActivitySerDeTest.java | 112 +++++++++++++++
.../processor/GooglePlusTypeConverterTest.java | 63 +++++---
.../resources/google_plus_activity_jsons.txt | 5 +
8 files changed, 434 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c921f674/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
index 928eec9..b4cf21d 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
@@ -58,10 +58,13 @@ public class GooglePlusTypeConverter implements StreamsProcessor {
LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
Activity activity = null;
+
if(item instanceof Person) {
activity = new Activity();
-
googlePlusActivityUtil.updateActivity((Person)item, activity);
+ } else if(item instanceof com.google.api.services.plus.model.Activity) {
+ activity = new Activity();
+ googlePlusActivityUtil.updateActivity((com.google.api.services.plus.model.Activity)item, activity);
}
if(activity != null) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c921f674/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
index 8eaa90a..1659ae3 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
@@ -18,42 +18,25 @@
package com.google.gplus.provider;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
import org.apache.commons.lang.NotImplementedException;
import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.json.*;
-import org.joda.time.DateTime;
+import org.apache.streams.pojo.json.Activity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Date;
import java.util.List;
-/**
-* Created with IntelliJ IDEA.
-* User: mdelaet
-* Date: 9/30/13
-* Time: 9:24 AM
-* To change this template use File | Settings | File Templates.
-*/
+
public class GPlusActivitySerializer implements ActivitySerializer<com.google.api.services.plus.model.Activity> {
private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivitySerializer.class);
GPlusProvider provider;
- ObjectMapper mapper = new ObjectMapper();
-
public GPlusActivitySerializer(GPlusProvider provider) {
this.provider = provider;
-
- mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, Boolean.FALSE);
-
}
public GPlusActivitySerializer() {
@@ -71,34 +54,9 @@ public class GPlusActivitySerializer implements ActivitySerializer<com.google.ap
@Override
public Activity deserialize(com.google.api.services.plus.model.Activity gplusActivity) {
-
- // There is totally a better way to do this
- // 1) Deep copy all jackson fields that overlap
- // 2) Check all objects are present
- // 3) Check essential fields have values
- // 4) Any that don't, set them based on other fields that are present
-
Activity activity = new Activity();
- activity.setId(formatId(gplusActivity.getId()));
- activity.setPublished(new DateTime(gplusActivity.getPublished().getValue()));
- Provider provider = new Provider();
- provider.setId("http://plus.google.com");
- provider.setDisplayName("GPlus");
- activity.setProvider(provider);
- Actor actor = new Actor();
- actor.setId(gplusActivity.getActor().getId());
- actor.setDisplayName(gplusActivity.getActor().getDisplayName());
- actor.setUrl(gplusActivity.getActor().getUrl());
- activity.setActor(actor);
- activity.setVerb(gplusActivity.getVerb());
- ActivityObject object = new ActivityObject();
- object.setId(gplusActivity.getObject().getId());
- object.setUrl(gplusActivity.getObject().getUrl());
- object.setContent(gplusActivity.getObject().getContent());
- activity.setTitle(gplusActivity.getTitle());
- activity.setContent(gplusActivity.getObject().getContent());
- activity.setObject(object);
+ GooglePlusActivityUtil.updateActivity(gplusActivity, activity);
return activity;
}
@@ -106,31 +64,4 @@ public class GPlusActivitySerializer implements ActivitySerializer<com.google.ap
public List<Activity> deserializeAll(List<com.google.api.services.plus.model.Activity> serializedList) {
throw new NotImplementedException("Not currently implemented");
}
-
- public static Generator buildGenerator(ObjectNode event) {
- return null;
- }
-
- public static Icon getIcon(ObjectNode event) {
- return null;
- }
-
- public static Provider buildProvider(ObjectNode event) {
- Provider provider = new Provider();
- provider.setId("id:providers:gmail");
- return provider;
- }
-
- public static List<Object> getLinks(ObjectNode event) {
- return null;
- }
-
- public static String getUrls(ObjectNode event) {
- return null;
- }
-
- public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:gmail", idparts));
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c921f674/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusEventProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusEventProcessor.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusEventProcessor.java
index 81cac86..6ed2ae1 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusEventProcessor.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusEventProcessor.java
@@ -29,9 +29,6 @@ import java.util.Queue;
import java.util.Random;
import java.util.concurrent.BlockingQueue;
-/**
- * Created by sblackmon on 12/10/13.
- */
public class GPlusEventProcessor implements Runnable {
private final static Logger LOGGER = LoggerFactory.getLogger(GPlusEventProcessor.class);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c921f674/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
new file mode 100644
index 0000000..9cb1d61
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.google.gplus.serializer.util;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.api.client.util.DateTime;
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Activity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Custom deserializer for GooglePlus' Person model
+ */
+public class GPlusActivityDeserializer extends JsonDeserializer<Activity> {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GPlusActivityDeserializer.class);
+
+ /**
+ * Because the GooglePlus Activity object {@link com.google.api.services.plus.model.Activity} contains complex objects
+ * within its hierarchy, we have to use a custom deserializer
+ *
+ * @param jsonParser
+ * @param deserializationContext
+ * @return The deserialized {@link com.google.api.services.plus.model.Activity} object
+ * @throws IOException
+ * @throws JsonProcessingException
+ */
+ @Override
+ public Activity deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+
+ JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+ Activity activity = new Activity();
+
+ try {
+ activity.setUrl(node.get("url").asText());
+ activity.setEtag(node.get("etag").asText());
+ activity.setTitle(node.get("title").asText());
+ activity.setPublished(DateTime.parseRfc3339(node.get("published").asText()));
+ activity.setUpdated(DateTime.parseRfc3339(node.get("updated").asText()));
+ activity.setId(node.get("id").asText());
+ activity.setVerb(node.get("verb").asText());
+
+ Activity.Actor actor = new Activity.Actor();
+ JsonNode actorNode = node.get("actor");
+
+ actor.setId(actorNode.get("id").asText());
+ actor.setDisplayName(actorNode.get("displayName").asText());
+ actor.setUrl(actorNode.get("url").asText());
+
+ Activity.Actor.Image image = new Activity.Actor.Image();
+ JsonNode imageNode = actorNode.get("image");
+ image.setUrl(imageNode.get("url").asText());
+
+ actor.setImage(image);
+ activity.setActor(actor);
+
+ Activity.PlusObject object = new Activity.PlusObject();
+ JsonNode objectNode = node.get("object");
+ object.setObjectType(objectNode.get("objectType").asText());
+ object.setContent(objectNode.get("content").asText());
+ object.setUrl(objectNode.get("url").asText());
+
+ Activity.PlusObject.Replies replies = new Activity.PlusObject.Replies();
+ JsonNode repliesNode = objectNode.get("replies");
+ replies.setTotalItems(repliesNode.get("totalItems").asLong());
+ replies.setSelfLink(repliesNode.get("selfLink").asText());
+ object.setReplies(replies);
+
+ Activity.PlusObject.Plusoners plusoners = new Activity.PlusObject.Plusoners();
+ JsonNode plusonersNode = objectNode.get("plusoners");
+ plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
+ plusoners.setSelfLink(plusonersNode.get("selfLink").asText());
+ object.setPlusoners(plusoners);
+
+ Activity.PlusObject.Resharers resharers = new Activity.PlusObject.Resharers();
+ JsonNode resharersNode = objectNode.get("resharers");
+ resharers.setTotalItems(resharersNode.get("totalItems").asLong());
+ resharers.setSelfLink(resharersNode.get("selfLink").asText());
+ object.setResharers(resharers);
+
+ List<Activity.PlusObject.Attachments> attachments = Lists.newArrayList();
+ for (JsonNode attachmentNode : objectNode.get("attachments")) {
+ Activity.PlusObject.Attachments attachments1 = new Activity.PlusObject.Attachments();
+ attachments1.setObjectType(attachmentNode.get("objectType").asText());
+ attachments1.setDisplayName(attachmentNode.get("displayName").asText());
+ attachments1.setContent(attachmentNode.get("content").asText());
+ attachments1.setUrl(attachmentNode.get("url").asText());
+
+ Activity.PlusObject.Attachments.Image image1 = new Activity.PlusObject.Attachments.Image();
+ JsonNode imageNode1 = attachmentNode.get("image");
+ image1.setUrl(imageNode1.get("url").asText());
+ attachments1.setImage(image1);
+
+ attachments.add(attachments1);
+ }
+ object.setAttachments(attachments);
+
+ activity.setObject(object);
+ } catch (Exception e) {
+ LOGGER.error("Exception while trying to deserialize activity object: {}", e);
+ }
+
+ return activity;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c921f674/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
index 3fce398..73b4b0d 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
@@ -20,25 +20,26 @@
package com.google.gplus.serializer.util;
import com.google.api.services.plus.model.Person;
+import org.apache.streams.pojo.json.*;
+import org.apache.streams.pojo.json.Activity;
+import org.joda.time.DateTime;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
+
+import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
public class GooglePlusActivityUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusActivityUtil.class);
/**
- * Given a Person object and an activity, fill out the appropriate actor details
+ * Given a {@link com.google.api.services.plus.model.Person} object and an
+ * {@link org.apache.streams.pojo.json.Activity} object, fill out the appropriate details
*
* @param item
* @param activity
@@ -57,7 +58,133 @@ public class GooglePlusActivityUtil {
}
/**
- * Extract the relevant details from the passed in Person object and build
+ * Given a Google Plus {@link com.google.api.services.plus.model.Activity},
+ * convert that into an Activity streams formatted {@link org.apache.streams.pojo.json.Activity}
+ *
+ * @param gPlusActivity
+ * @param activity
+ */
+ public static void updateActivity(com.google.api.services.plus.model.Activity gPlusActivity, Activity activity) {
+ activity.setActor(buildActor(gPlusActivity.getActor()));
+ activity.setVerb("post");
+ activity.setTitle(gPlusActivity.getTitle());
+ activity.setUrl(gPlusActivity.getUrl());
+ activity.setProvider(getProvider());
+
+ if(gPlusActivity.getObject() != null) {
+ activity.setContent(gPlusActivity.getObject().getContent());
+ }
+
+ activity.setId(formatId(activity.getVerb(),
+ Optional.fromNullable(
+ gPlusActivity.getId())
+ .orNull()));
+
+ DateTime published = new DateTime(String.valueOf(gPlusActivity.getPublished()));
+ activity.setPublished(published);
+
+ setObject(activity, gPlusActivity.getObject());
+ addGPlusExtensions(activity, gPlusActivity);
+ }
+
+ /**
+ * Add in necessary extensions from the passed in {@link com.google.api.services.plus.model.Activity} to the
+ * {@link org.apache.streams.pojo.json.Activity} object
+ *
+ * @param activity
+ * @param gPlusActivity
+ */
+ private static void addGPlusExtensions(Activity activity, com.google.api.services.plus.model.Activity gPlusActivity) {
+ Map<String, Object> extensions = ensureExtensions(activity);
+
+ com.google.api.services.plus.model.Activity.PlusObject object = gPlusActivity.getObject();
+ extensions.put("googlePlus", gPlusActivity);
+
+ if(object != null) {
+ com.google.api.services.plus.model.Activity.PlusObject.Plusoners plusoners = object.getPlusoners();
+ if(plusoners != null) {
+ Map<String, Object> likes = new HashMap<>();
+ likes.put("count", plusoners.getTotalItems());
+ extensions.put("likes", likes);
+ }
+
+ com.google.api.services.plus.model.Activity.PlusObject.Resharers resharers = object.getResharers();
+ if(resharers != null) {
+ Map<String, Object> rebroadcasts = new HashMap<>();
+ rebroadcasts.put("count", resharers.getTotalItems());
+ extensions.put("rebroadcasts", rebroadcasts);
+ }
+
+ extensions.put("keywords", object.getContent());
+ }
+ }
+
+ /**
+ * Set the {@link org.apache.streams.pojo.json.ActivityObject} field given the passed in
+ * {@link com.google.api.services.plus.model.Activity.PlusObject}
+ *
+ * @param activity
+ * @param object
+ */
+ private static void setObject(Activity activity, com.google.api.services.plus.model.Activity.PlusObject object) {
+ if(object != null) {
+ ActivityObject activityObject = new ActivityObject();
+
+ activityObject.setContent(object.getContent());
+ activityObject.setObjectType(object.getObjectType());
+
+ java.util.List<ActivityObject> attachmentsList = Lists.newArrayList();
+ for (com.google.api.services.plus.model.Activity.PlusObject.Attachments attachments : object.getAttachments()) {
+ ActivityObject attach = new ActivityObject();
+
+ attach.setContent(attachments.getContent());
+ attach.setDisplayName(attachments.getDisplayName());
+ attach.setObjectType(attachments.getObjectType());
+ attach.setUrl(attachments.getUrl());
+
+ Image image = new Image();
+ com.google.api.services.plus.model.Activity.PlusObject.Attachments.Image image1 = attachments.getImage();
+
+ if (image1 != null) {
+ image.setUrl(image1.getUrl());
+ attach.setImage(image);
+ }
+
+ attachmentsList.add(attach);
+ }
+
+ activityObject.setAttachments(attachmentsList);
+
+ activity.setObject(activityObject);
+ }
+ }
+
+ /**
+ * Given a {@link com.google.api.services.plus.model.Activity.Actor} object, return a fully fleshed
+ * out {@link org.apache.streams.pojo.json.Actor} object
+ *
+ * @param gPlusActor
+ * @return
+ */
+ private static Actor buildActor(com.google.api.services.plus.model.Activity.Actor gPlusActor) {
+ Actor actor = new Actor();
+
+ actor.setDisplayName(gPlusActor.getDisplayName());
+ actor.setId(formatId(String.valueOf(gPlusActor.getId())));
+ actor.setUrl(gPlusActor.getUrl());
+
+ Image image = new Image();
+ com.google.api.services.plus.model.Activity.Actor.Image googlePlusImage = gPlusActor.getImage();
+
+ if(googlePlusImage != null) {
+ image.setUrl(googlePlusImage.getUrl());
+ }
+ actor.setImage(image);
+
+ return actor;
+ }
+ /**
+ * Extract the relevant details from the passed in {@link com.google.api.services.plus.model.Person} object and build
* an actor with them
*
* @param person
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c921f674/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusActivitySerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusActivitySerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusActivitySerDeTest.java
new file mode 100644
index 0000000..6babea0
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusActivitySerDeTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.google.gplus.processor;
+
+import com.fasterxml.jackson.databind.*;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.gplus.serializer.util.GPlusActivityDeserializer;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.apache.streams.pojo.json.Provider;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class GooglePlusActivitySerDeTest {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusActivitySerDeTest.class);
+ private ObjectMapper objectMapper;
+ private GooglePlusActivityUtil googlePlusActivityUtil;
+
+ @Before
+ public void setup() {
+ objectMapper = new StreamsJacksonMapper();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ googlePlusActivityUtil = new GooglePlusActivityUtil();
+ }
+
+ @Test
+ public void TestActivityObjects() {
+ InputStream is = GooglePlusActivitySerDeTest.class.getResourceAsStream("/google_plus_activity_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Activity activity = new Activity();
+
+ com.google.api.services.plus.model.Activity gPlusActivity = objectMapper.readValue(line, com.google.api.services.plus.model.Activity.class);
+
+ googlePlusActivityUtil.updateActivity(gPlusActivity, activity);
+ LOGGER.info("activity: {}", activity);
+
+ assertNotNull(activity);
+ assert(activity.getId().contains("id:googleplus:post"));
+ assertEquals(activity.getVerb(), "post");
+
+ Provider provider = activity.getProvider();
+ assertEquals(provider.getId(), "id:providers:googleplus");
+ assertEquals(provider.getDisplayName(), "GooglePlus");
+
+ Actor actor = activity.getActor();
+ assertNotNull(actor.getImage());
+ assert(actor.getId().contains("id:googleplus:"));
+ assertNotNull(actor.getUrl());
+
+ assertNotNull(activity.getPublished());
+ assertNotNull(activity.getTitle());
+ assertNotNull(activity.getUrl());
+
+ Map<String, Object> extensions = (Map<String, Object>)activity.getAdditionalProperties().get("extensions");
+ assertNotNull(extensions);
+ assertNotNull(extensions.get("googlePlus"));
+
+ if(activity.getContent() != null) {
+ assertNotNull(extensions.get("rebroadcasts"));
+ assertNotNull(extensions.get("keywords"));
+ assertNotNull(extensions.get("likes"));
+ assert (((Map<String, Object>) extensions.get("rebroadcasts")).containsKey("count"));
+ assert (((Map<String, Object>) extensions.get("likes")).containsKey("count"));
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.error("Exception while testing serializability: {}", e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c921f674/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
index 6fbdf19..a8a44c0 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
@@ -22,10 +22,12 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.api.services.plus.model.Person;
+import com.google.gplus.serializer.util.GPlusActivityDeserializer;
import com.google.gplus.serializer.util.GPlusPersonDeserializer;
import com.google.gplus.serializer.util.GooglePlusActivityUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.exceptions.ActivitySerializerException;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.Activity;
import org.junit.Before;
@@ -34,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
@@ -51,6 +54,7 @@ public class GooglePlusTypeConverterTest {
objectMapper = new StreamsJacksonMapper();
SimpleModule simpleModule = new SimpleModule();
simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
+ simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
objectMapper.registerModule(simpleModule);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
@@ -59,33 +63,56 @@ public class GooglePlusTypeConverterTest {
}
@Test
- public void testProcess() {
+ public void testProcessPerson() throws IOException, ActivitySerializerException {
InputStream is = GooglePlusTypeConverterTest.class.getResourceAsStream("/google_plus_person_jsons.txt");
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Activity activity = new Activity();
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Activity activity = new Activity();
- Person person = objectMapper.readValue(line, Person.class);
- StreamsDatum streamsDatum = new StreamsDatum(person);
+ Person person = objectMapper.readValue(line, Person.class);
+ StreamsDatum streamsDatum = new StreamsDatum(person);
- assertNotNull(streamsDatum.getDocument());
+ assertNotNull(streamsDatum.getDocument());
- List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
- GooglePlusActivityUtil.updateActivity(person, activity);
+ List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
+ GooglePlusActivityUtil.updateActivity(person, activity);
- assertEquals(retList.size(), 1);
- assert(retList.get(0).getDocument() instanceof Activity);
- assertEquals(activity, retList.get(0).getDocument());
- }
+ assertEquals(retList.size(), 1);
+ assert(retList.get(0).getDocument() instanceof Activity);
+ assertEquals(activity, retList.get(0).getDocument());
+ }
+ }
+ }
+
+ @Test
+ public void testProcessActivity() throws IOException, ActivitySerializerException{
+ InputStream is = GooglePlusTypeConverterTest.class.getResourceAsStream("/google_plus_activity_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Activity activity = new Activity();
+
+ com.google.api.services.plus.model.Activity gPlusActivity = objectMapper.readValue(line, com.google.api.services.plus.model.Activity.class);
+ StreamsDatum streamsDatum = new StreamsDatum(gPlusActivity);
+
+ assertNotNull(streamsDatum.getDocument());
+
+ List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
+ GooglePlusActivityUtil.updateActivity(gPlusActivity, activity);
+
+ assertEquals(retList.size(), 1);
+ assert(retList.get(0).getDocument() instanceof Activity);
+ assertEquals(activity, retList.get(0).getDocument());
}
- } catch (Exception e) {
- LOGGER.error("Exception testing the GooglePlusTypeConverter: {}", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c921f674/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_activity_jsons.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_activity_jsons.txt b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_activity_jsons.txt
new file mode 100644
index 0000000..079251b
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_activity_jsons.txt
@@ -0,0 +1,5 @@
+{ "kind": "plus#activity", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/NWJTEjOXM_BKuKTTfM8bvWt5c0E\"", "title": "", "published": "2014-10-08T15:53:52.703Z", "updated": "2014-10-08T15:53:52.703Z", "id": "z130ipoawvulthecp22otlwonv2gutmnm", "url": "https://plus.google.com/118376908737486995861/posts/4Zr85DnfdsV", "actor": { "id": "118376908737486995861", "displayName": "Steve Blackmon", "url": "https://plus.google.com/118376908737486995861", "image": { "url": "https://lh3.googleusercontent.com/-XdUIqdMkCWA/AAAAAAAAAAI/AAAAAAAAAAA/4252rscbv5M/photo.jpg?sz=50" } }, "verb": "post", "object": { "objectType": "note", "content": "", "url": "https://plus.google.com/118376908737486995861/posts/4Zr85DnfdsV", "replies": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z130ipoawvulthecp22otlwonv2gutmnm/comments" }, "plusoners": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z130ipoawvulthecp22otlwonv2gutmnm/people/plusoners" }, "resh
arers": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z130ipoawvulthecp22otlwonv2gutmnm/people/resharers" }, "attachments": [ { "objectType": "article", "displayName": "Diction in Software Development (i.e. Don't be a d1ck!)", "content": "Over the years, I've come to realize how important diction is in software development (and life in general). It may mean the difference between a 15 minute meeting where everyone nods their heads, and a day long battle of eg...", "url": "http://brianoneill.blogspot.com/2014/10/diction-in-software-development-ie-dont.html", "image": { "url": "https://lh3.googleusercontent.com/proxy/3BTfJNati40qZ5zdoAK1Zc-udwAMRqYbPX6huwDSHU3HXW9ZlJ2OIaWtoAG3ta1rSy5X=w120-h120", "type": "image/jpeg", "height": 120, "width": 120 }, "fullImage": { "url": "http://www.mindrain.com/fishing.jpg", "type": "image/jpeg" } } ] }, "provider": { "title": "Google+" }, "access": { "kind": "plus#acl", "description": "Public", "items": [ { "type":
"public" } ] } }
+{ "kind": "plus#activity", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/ZRTkaAm2NktNymi4G6PNueqVXV0\"", "title": "I have almost all positive thoughts on this phone. Minor concerns on looks (not that important I know...", "published": "2014-10-28T16:27:31.774Z", "updated": "2014-10-28T16:27:31.774Z", "id": "z12pwxry0qapdj5wr04cedb4bs20ilg55zk0k", "url": "https://plus.google.com/116771159471120611293/posts/SHeUgMcXLpz", "actor": { "id": "116771159471120611293", "displayName": "Matt Neithercott", "url": "https://plus.google.com/116771159471120611293", "image": { "url": "https://lh6.googleusercontent.com/-C0fiZBxdvw0/AAAAAAAAAAI/AAAAAAAAJ5k/K4pgR3_-_ms/photo.jpg?sz=50" } }, "verb": "post", "object": { "objectType": "note", "content": "I have almost all positive thoughts on this phone. Minor concerns on looks (not that important I know, but I like the Moto X/Nexus 6 look) and more importantly I'm wondering if they will be as good with the Droid line with Android updates... those seem to ha
ve lagged a bit more for this line in the past. Mainly I want Android Lollipop. <br /><br />Over all this phone looks like a beast and easily one of the best phones on the market. Oh, can we talk about the battery?!? \ufeff", "url": "https://plus.google.com/116771159471120611293/posts/SHeUgMcXLpz", "replies": { "totalItems": 2, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12pwxry0qapdj5wr04cedb4bs20ilg55zk0k/comments" }, "plusoners": { "totalItems": 3, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12pwxry0qapdj5wr04cedb4bs20ilg55zk0k/people/plusoners" }, "resharers": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12pwxry0qapdj5wr04cedb4bs20ilg55zk0k/people/resharers" }, "attachments": [ { "objectType": "video", "displayName": "Motorola Droid Turbo hands-on", "content": "Motorola's partnership with Verizon continues with the Droid Turbo, a unique smartphone exclusive to Verizon Wireless. Subscribe: http://www.yo
utube.com/subs...", "url": "http://www.youtube.com/attribution_link?a=k_fPNZUvh50&u=/watch?v%3DVo3jj4GvEG0%26feature%3Dshare", "image": { "url": "https://lh3.googleusercontent.com/proxy/1MR0ExMhC2wxZj_g0dMnChwzNkg9-eC9N3VVfdfsBAuQZ1Li9U-kZoqPJ8N1YD452vASBH6_6Meh90GyrK1cSdXqZw=w506-h284-n", "type": "image/jpeg", "height": 284, "width": 506 }, "embed": { "url": "https://www.youtube.com/embed/Vo3jj4GvEG0", "type": "application/x-shockwave-flash" } } ] }, "provider": { "title": "Google+" }, "access": { "kind": "plus#acl", "description": "Public", "items": [ { "type": "public" } ] } }
+{ "kind": "plus#activity", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/U-nMv_RITNj07a6rnKIZ4z6Dk4k\"", "title": "Truth hurts...", "published": "2014-10-28T15:36:47.423Z", "updated": "2014-10-28T15:36:47.423Z", "id": "z12nwnsbkoivjjwvt22mdt1y0k3of1djw04", "url": "https://plus.google.com/116771159471120611293/posts/MPe25pfK4hU", "actor": { "id": "116771159471120611293", "displayName": "Matt Neithercott", "url": "https://plus.google.com/116771159471120611293", "image": { "url": "https://lh6.googleusercontent.com/-C0fiZBxdvw0/AAAAAAAAAAI/AAAAAAAAJ5k/K4pgR3_-_ms/photo.jpg?sz=50" } }, "verb": "share", "object": { "objectType": "activity", "id": "z13tv1wasli0ypjih04cfpk4dti3yrhjawo0k", "actor": { "id": "109198125327200245055", "displayName": "Stephani Renteria", "url": "https://plus.google.com/109198125327200245055", "image": { "url": "https://lh4.googleusercontent.com/-WJeeS-ny6Ac/AAAAAAAAAAI/AAAAAAAAB5I/zdPCWdOGQuQ/photo.jpg?sz=50" } }, "content": "Truth hurts...\ufeff", "url": "https://plus.
google.com/109198125327200245055/posts/bNSgdkCPmbG", "replies": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12nwnsbkoivjjwvt22mdt1y0k3of1djw04/comments" }, "plusoners": { "totalItems": 2, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12nwnsbkoivjjwvt22mdt1y0k3of1djw04/people/plusoners" }, "resharers": { "totalItems": 1, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12nwnsbkoivjjwvt22mdt1y0k3of1djw04/people/resharers" }, "attachments": [ { "objectType": "photo", "displayName": "Truth hurts...", "id": "109198125327200245055.6075076220744786466", "content": "16/9/14 - 1", "url": "https://plus.google.com/photos/109198125327200245055/albums/6075076218164640065/6075076220744786466", "image": { "url": "https://lh4.googleusercontent.com/-WjV4kLfSUjo/VE8BorjqYiI/AAAAAAAAC2g/D46Yfy7xO_U/w506-h750/14%2B-%2B1", "type": "image/jpeg" }, "fullImage": { "url": "https://lh4.googleusercontent.com/-WjV4kLfSUjo/VE8BorjqYiI/AAAAA
AAAC2g/D46Yfy7xO_U/w476-h626/14%2B-%2B1", "type": "image/jpeg", "height": 626, "width": 476 } } ] }, "provider": { "title": "Reshared Post" }, "access": { "kind": "plus#acl", "description": "Public", "items": [ { "type": "public" } ] } }
+{ "kind": "plus#activity", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/Syi_EzWY7OiXcsP6iOpMGjnQnFk\"", "title": "The Droid Turbo might be hard to say no to... ", "published": "2014-10-28T02:44:40.380Z", "updated": "2014-10-28T02:44:40.380Z", "id": "z12bzbxrmlzoxvzl222mdt1y0k3of1djw04", "url": "https://plus.google.com/116771159471120611293/posts/6rSySsYPAVL", "actor": { "id": "116771159471120611293", "displayName": "Matt Neithercott", "url": "https://plus.google.com/116771159471120611293", "image": { "url": "https://lh6.googleusercontent.com/-C0fiZBxdvw0/AAAAAAAAAAI/AAAAAAAAJ5k/K4pgR3_-_ms/photo.jpg?sz=50" } }, "verb": "post", "object": { "objectType": "note", "content": "The Droid Turbo might be hard to say no to... \ufeff", "url": "https://plus.google.com/116771159471120611293/posts/6rSySsYPAVL", "replies": { "totalItems": 4, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12bzbxrmlzoxvzl222mdt1y0k3of1djw04/comments" }, "plusoners": { "totalItems": 1, "selfLink": "https:
//content.googleapis.com/plus/v1/activities/z12bzbxrmlzoxvzl222mdt1y0k3of1djw04/people/plusoners" }, "resharers": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12bzbxrmlzoxvzl222mdt1y0k3of1djw04/people/resharers" } }, "provider": { "title": "Google+" }, "access": { "kind": "plus#acl", "description": "Public", "items": [ { "type": "public" } ] } }, { "kind": "plus#activity", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/rQfbBZAN11iZKWdIm2RX3g2kIys\"", "title": "", "published": "2014-10-28T01:13:08.582Z", "updated": "2014-10-28T01:13:08.582Z", "id": "z12itxgadmqsfpjut22mdt1y0k3of1djw04", "url": "https://plus.google.com/116771159471120611293/posts/FyDp4gDoxya", "actor": { "id": "116771159471120611293", "displayName": "Matt Neithercott", "url": "https://plus.google.com/116771159471120611293", "image": { "url": "https://lh6.googleusercontent.com/-C0fiZBxdvw0/AAAAAAAAAAI/AAAAAAAAJ5k/K4pgR3_-_ms/photo.jpg?sz=50" } }, "verb": "post", "object": { "objectType": "
note", "content": "", "url": "https://plus.google.com/116771159471120611293/posts/FyDp4gDoxya", "replies": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12itxgadmqsfpjut22mdt1y0k3of1djw04/comments" }, "plusoners": { "totalItems": 2, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12itxgadmqsfpjut22mdt1y0k3of1djw04/people/plusoners" }, "resharers": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z12itxgadmqsfpjut22mdt1y0k3of1djw04/people/resharers" }, "attachments": [ { "objectType": "article", "displayName": "iPhone fans and Android fans finally find something they hate more than each other", "content": "It looks like Walmart, CVS and other merchants have stepped into a major hornets' nest. In the wake of news that several major retailers are turning off NFC payment sensors in their stores because ...", "url": "http://bgr.com/2014/10/27/apple-pay-google-wallet-vs-currentc/", "image": { "url": "htt
ps://lh6.googleusercontent.com/proxy/6iMaeS_62HVihJjR77NQXujzcoGwSkSZUsafEp3DJiSyyvp7VyT5KU8-_o6zL3DpXGAjbvAJv3GnEIyKG6gYGg1SU6eCfbYX=w506-h303-p", "type": "image/jpeg", "height": 303, "width": 506 }, "fullImage": { "url": "http://cdn.bgr.com/2012/09/androidios-e1346785636929.jpg", "type": "image/jpeg" } } ] }, "provider": { "title": "Google+" }, "access": { "kind": "plus#acl", "description": "Public", "items": [ { "type": "public" } ] } }
+{ "kind": "plus#activity", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/v1-6aVSBGT4qiStMoz7f2_AN2fM\"", "title": "", "published": "2014-10-27T06:26:33.927Z", "updated": "2014-10-27T06:26:33.927Z", "id": "z13twrlznpvtzz52w22mdt1y0k3of1djw04", "url": "https://plus.google.com/116771159471120611293/posts/GR7CGR8N5VL", "actor": { "id": "116771159471120611293", "displayName": "Matt Neithercott", "url": "https://plus.google.com/116771159471120611293", "image": { "url": "https://lh6.googleusercontent.com/-C0fiZBxdvw0/AAAAAAAAAAI/AAAAAAAAJ5k/K4pgR3_-_ms/photo.jpg?sz=50" } }, "verb": "share", "object": { "objectType": "activity", "id": "z13zgvtiurjgfti1v234iflghvq2c1dge04", "actor": { "id": "104954254300557350002", "displayName": "Adam Balm", "url": "https://plus.google.com/104954254300557350002", "image": { "url": "https://lh4.googleusercontent.com/-SO1scj4p2LA/AAAAAAAAAAI/AAAAAAAAI-s/efA9LBVe144/photo.jpg?sz=50" } }, "content": "", "url": "https://plus.google.com/104954254300557350002/posts/AwewX
htn7ws", "replies": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/comments" }, "plusoners": { "totalItems": 9, "selfLink": "https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/people/plusoners" }, "resharers": { "totalItems": 0, "selfLink": "https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/people/resharers" }, "attachments": [ { "objectType": "photo", "id": "104954254300557350002.6074732746360957410", "content": "26/10/2014 - 1", "url": "https://plus.google.com/photos/104954254300557350002/albums/6074732747132702225/6074732746360957410", "image": { "url": "https://lh4.googleusercontent.com/-oO3fnARlDm0/VE3JP1xHKeI/AAAAAAAAeCY/-X2jzc6HruA/w506-h750/2014%2B-%2B1", "type": "image/jpeg" }, "fullImage": { "url": "https://lh4.googleusercontent.com/-oO3fnARlDm0/VE3JP1xHKeI/AAAAAAAAeCY/-X2jzc6HruA/w600-h1141/2014%2B-%2B1", "type": "image/jpeg", "heig
ht": 1141, "width": 600 } } ] }, "annotation": "Truth 😜", "provider": { "title": "Reshared Post" }, "access": { "kind": "plus#acl", "description": "Public", "items": [ { "type": "public" } ] } }
\ No newline at end of file
[06/38] incubator-streams git commit: region should be optional.
defaults to us-east-1
Posted by sb...@apache.org.
region should be optional. defaults to us-east-1
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/8088ac98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/8088ac98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/8088ac98
Branch: refs/heads/STREAMS-49
Commit: 8088ac983d047436f795b2df37c6a26a6e50e7f2
Parents: 25af3ad
Author: sblackmon <sb...@apache.org>
Authored: Tue Oct 7 13:55:41 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Oct 7 13:55:41 2014 -0500
----------------------------------------------------------------------
.../src/main/jsonschema/org/apache/streams/s3/S3Configuration.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8088ac98/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
index 36e89d0..a136308 100644
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
+++ b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
@@ -29,7 +29,7 @@
"region": {
"type": "string",
"description": "The AWS region where your bucket resides",
- "required": true
+ "required": false
}
}
}
\ No newline at end of file
[29/38] incubator-streams git commit: STREAMS-204 | Removed file with
typo in name
Posted by sb...@apache.org.
STREAMS-204 | Removed file with typo in name
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e792bb9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e792bb9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e792bb9e
Branch: refs/heads/STREAMS-49
Commit: e792bb9e1ef4d9c9257acd6e97bca466511f6d0e
Parents: 08e2145
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Thu Nov 6 14:10:17 2014 -0600
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Thu Nov 6 14:12:03 2014 -0600
----------------------------------------------------------------------
.../gplus/GooglePLusCommentSerDeTest.java | 114 -------------------
1 file changed, 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e792bb9e/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
deleted file mode 100644
index 9fea22c..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.client.util.Lists;
-import com.google.api.services.plus.model.Comment;
-import com.google.gplus.serializer.util.GPlusCommentDeserializer;
-import com.google.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.commons.lang.StringUtils;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
-public class GooglePlusCommentSerDeTest {
- private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentSerDeTest.class);
- private ObjectMapper objectMapper;
- private GooglePlusActivityUtil googlePlusActivityUtil;
-
- @Before
- public void setup() {
- objectMapper = new StreamsJacksonMapper();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Comment.class, new GPlusCommentDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- googlePlusActivityUtil = new GooglePlusActivityUtil();
- }
-
- @org.junit.Test
- public void testCommentObjects() {
- InputStream is = GooglePlusPersonSerDeTest.class.getResourceAsStream("/google_plus_comments_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- Activity activity = new Activity();
- List<Comment> comments = Lists.newArrayList();
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Comment comment = objectMapper.readValue(line, Comment.class);
-
- LOGGER.info("comment: {}", comment);
-
- assertNotNull(comment);
- assertNotNull(comment.getEtag());
- assertNotNull(comment.getId());
- assertNotNull(comment.getInReplyTo());
- assertNotNull(comment.getObject());
- assertNotNull(comment.getPlusoners());
- assertNotNull(comment.getPublished());
- assertNotNull(comment.getUpdated());
- assertNotNull(comment.getSelfLink());
- assertEquals(comment.getVerb(), "post");
-
- comments.add(comment);
- }
- }
-
- assertEquals(comments.size(), 3);
-
- googlePlusActivityUtil.updateActivity(comments, activity);
- assertNotNull(activity);
- assertNotNull(activity.getObject());
- assertEquals(activity.getObject().getAttachments().size(), 3);
- } catch (Exception e) {
- LOGGER.error("Exception while testing serializability: {}", e);
- }
- }
-
- @org.junit.Test
- public void testEmptyComments() {
- Activity activity = new Activity();
-
- googlePlusActivityUtil.updateActivity(new ArrayList<Comment>(), activity);
-
- assertNull(activity.getObject());
- }
-}
[37/38] incubator-streams git commit: Merge branch 'STREAMS-208' into
STREAMS-49
Posted by sb...@apache.org.
Merge branch 'STREAMS-208' into STREAMS-49
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/073b0830
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/073b0830
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/073b0830
Branch: refs/heads/STREAMS-49
Commit: 073b083000b365da2b850ea5b6d9da3f29a06770
Parents: 409b448 adbb1b8
Author: sblackmon <sb...@apache.org>
Authored: Fri Nov 7 11:54:02 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Fri Nov 7 11:54:02 2014 -0800
----------------------------------------------------------------------
pom.xml | 56 +++++
streams-contrib/streams-amazon-aws/pom.xml | 2 +-
.../org/apache/streams/s3/S3Configurator.java | 55 ++---
.../org/apache/streams/s3/S3PersistReader.java | 11 +-
.../org/apache/streams/s3/S3PersistWriter.java | 10 +-
.../org/apache/streams/s3/S3Configuration.json | 18 +-
streams-contrib/streams-persist-console/pom.xml | 34 +++
.../streams/console/ConsolePersistReader.java | 11 +-
.../streams/console/ConsolePersistWriter.java | 11 +-
.../processor/PercolateTagProcessor.java | 10 +-
.../ElasticsearchWriterConfiguration.json | 10 +
.../google-gplus/pom.xml | 2 +-
.../processor/GooglePlusTypeConverter.java | 94 +++++++
.../gplus/provider/GPlusActivitySerializer.java | 77 +-----
.../gplus/provider/GPlusEventProcessor.java | 3 -
.../util/GPlusActivityDeserializer.java | 162 +++++++++++++
.../util/GPlusPersonDeserializer.java | 107 ++++++++
.../serializer/util/GooglePlusActivityUtil.java | 242 +++++++++++++++++++
.../google/gplus/GooglePlusPersonSerDeTest.java | 97 ++++++++
.../processor/GooglePlusActivitySerDeTest.java | 112 +++++++++
.../processor/GooglePlusTypeConverterTest.java | 125 ++++++++++
.../resources/google_plus_activity_jsons.txt | 5 +
.../test/resources/google_plus_person_jsons.txt | 2 +
23 files changed, 1125 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
[18/38] incubator-streams git commit: resolves STREAMS-206,
pending feedback
Posted by sb...@apache.org.
resolves STREAMS-206, pending feedback
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/1ee614e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/1ee614e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/1ee614e7
Branch: refs/heads/STREAMS-49
Commit: 1ee614e77d8fa16cc58aec689d5d7716a8e29026
Parents: eec3aa9
Author: sblackmon <sb...@apache.org>
Authored: Thu Oct 30 11:48:15 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Thu Oct 30 11:48:15 2014 -0500
----------------------------------------------------------------------
.../elasticsearch/processor/PercolateTagProcessor.java | 10 ++++++----
.../elasticsearch/ElasticsearchWriterConfiguration.json | 10 ++++++++++
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1ee614e7/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessor.java
index 0b62ce9..ca9be8d 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessor.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessor.java
@@ -202,9 +202,11 @@ public class PercolateTagProcessor implements StreamsProcessor {
manager = new ElasticsearchClientManager(config);
bulkBuilder = manager.getClient().prepareBulk();
createIndexIfMissing(config.getIndex());
- deleteOldQueries(config.getIndex());
+ if( config.getReplaceTags() == true ) {
+ deleteOldQueries(config.getIndex());
+ }
for (String tag : config.getTags().getAdditionalProperties().keySet()) {
- String query = (String)config.getTags().getAdditionalProperties().get(tag);
+ String query = (String) config.getTags().getAdditionalProperties().get(tag);
PercolateQueryBuilder queryBuilder = new PercolateQueryBuilder(tag, query, this.usePercolateField);
addPercolateRule(queryBuilder, config.getIndex());
}
@@ -213,12 +215,12 @@ public class PercolateTagProcessor implements StreamsProcessor {
else
LOGGER.error("FAILED writing " + bulkBuilder.numberOfActions() + " tags to " + config.getIndex() + " _percolator");
-
}
@Override
public void cleanUp() {
- deleteOldQueries(config.getIndex());
+ if( config.getCleanupTags() == true )
+ deleteOldQueries(config.getIndex());
manager.getClient().close();
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1ee614e7/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
index 13017e2..603b0bf 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
@@ -41,6 +41,16 @@
"javaInterfaces": ["java.io.Serializable"],
"description": "Tags to apply during index"
},
+ "replaceTags": {
+ "type": "boolean",
+ "description": "Whether to overwrite _percolator rules on startup",
+ "default": true
+ },
+ "cleanupTags": {
+ "type": "boolean",
+ "description": "Whether to remove _percolator rules on shutdown",
+ "default": true
+ },
"forceUseConfig": {
"type": "boolean",
"description": "Whether or not we force the values that are set in the configuration"
[19/38] incubator-streams git commit: Merge remote-tracking branch
'origin/STREAMS-187'
Posted by sb...@apache.org.
Merge remote-tracking branch 'origin/STREAMS-187'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d88e8c88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d88e8c88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d88e8c88
Branch: refs/heads/STREAMS-49
Commit: d88e8c8896442d7f7b537f5159f6619e7a0f0a6c
Parents: 42d0ab3 8088ac9
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Oct 31 09:46:35 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Oct 31 09:46:35 2014 -0500
----------------------------------------------------------------------
streams-contrib/streams-amazon-aws/pom.xml | 2 +-
.../org/apache/streams/s3/S3Configurator.java | 55 +++++++++-----------
.../org/apache/streams/s3/S3PersistReader.java | 11 ++--
.../org/apache/streams/s3/S3PersistWriter.java | 10 ++--
.../org/apache/streams/s3/S3Configuration.json | 18 +++++--
5 files changed, 55 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
[35/38] incubator-streams git commit: Merge remote-tracking branch
'rbnks/STREAMS-199'
Posted by sb...@apache.org.
Merge remote-tracking branch 'rbnks/STREAMS-199'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/660f0c13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/660f0c13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/660f0c13
Branch: refs/heads/STREAMS-49
Commit: 660f0c13639003efd6a0355f25dc25ed66c4b8dc
Parents: b8e7a69 79eadea
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Nov 7 13:00:13 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Nov 7 13:00:13 2014 -0600
----------------------------------------------------------------------
.../local/builders/LocalStreamBuilder.java | 5 +++
.../streams/local/tasks/StreamsMergeTask.java | 7 +++++
.../local/tasks/StreamsPersistWriterTask.java | 18 ++++++++---
.../local/tasks/StreamsProcessorTask.java | 20 +++++++++---
.../local/tasks/StreamsProviderTask.java | 15 +++++++++
.../apache/streams/local/tasks/StreamsTask.java | 4 +++
.../local/builders/LocalStreamBuilderTest.java | 32 +++++++++++++++++---
.../streams/local/tasks/BasicTasksTest.java | 26 +++++++++++++---
8 files changed, 110 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[22/38] incubator-streams git commit: Fixed merge error
Posted by sb...@apache.org.
Fixed merge error
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/79eadea2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/79eadea2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/79eadea2
Branch: refs/heads/STREAMS-49
Commit: 79eadea2453610075236e1fed13c215bee474592
Parents: 377fb89
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Oct 31 17:00:47 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Oct 31 17:00:47 2014 -0500
----------------------------------------------------------------------
.../java/org/apache/streams/local/tasks/StreamsProcessorTask.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/79eadea2/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index 33c5827..b6ab498 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -127,6 +127,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
if(output != null) {
for(StreamsDatum outDatum : output) {
super.addToOutgoingQueue(outDatum);
+ this.counter.incrementEmittedCount();
statusCounter.incrementStatus(DatumStatus.SUCCESS);
}
}
[13/38] incubator-streams git commit: Fixed tests by unregistering
previously registered mbeans
Posted by sb...@apache.org.
Fixed tests by unregistering previously registered mbeans
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d71568dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d71568dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d71568dc
Branch: refs/heads/STREAMS-49
Commit: d71568dc72716d8d6b10ef0de74df5325f3d1336
Parents: 7e65a42
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Mon Oct 20 15:30:28 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Mon Oct 20 15:30:28 2014 -0500
----------------------------------------------------------------------
.../local/builders/LocalStreamBuilder.java | 5 +++
.../local/builders/LocalStreamBuilderTest.java | 32 +++++++++++++++++---
2 files changed, 32 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d71568dc/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 57f3aa4..2161638 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -20,6 +20,7 @@ package org.apache.streams.local.builders;
import org.apache.log4j.spi.LoggerFactory;
import org.apache.streams.core.*;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.streams.local.executors.ShutdownStreamOnUnhandleThrowableThreadPoolExecutor;
import org.apache.streams.local.queues.ThroughputQueue;
import org.apache.streams.local.tasks.LocalStreamProcessMonitorThread;
@@ -271,6 +272,8 @@ public class LocalStreamBuilder implements StreamBuilder {
for(StreamComponent prov : this.providers.values()) {
StreamsTask task = prov.createConnectedTask(getTimeout());
task.setStreamConfig(this.streamConfig);
+ StreamsTaskCounter counter = new StreamsTaskCounter(prov.getId());
+ task.setStreamsTaskCounter(counter);
this.executor.submit(task);
provTasks.put(prov.getId(), (StreamsProviderTask) task);
if( prov.isOperationCountable() ) {
@@ -284,8 +287,10 @@ public class LocalStreamBuilder implements StreamBuilder {
for(StreamComponent comp : this.components.values()) {
int tasks = comp.getNumTasks();
List<StreamsTask> compTasks = new LinkedList<StreamsTask>();
+ StreamsTaskCounter counter = new StreamsTaskCounter(comp.getId());
for(int i=0; i < tasks; ++i) {
StreamsTask task = comp.createConnectedTask(getTimeout());
+ task.setStreamsTaskCounter(counter);
task.setStreamConfig(this.streamConfig);
this.futures.put(task, this.executor.submit(task));
compTasks.add(task);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d71568dc/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
index e602181..fea7e53 100644
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
@@ -43,6 +43,7 @@ import org.apache.streams.core.StreamBuilder;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsPersistWriter;
import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.streams.local.queues.ThroughputQueue;
import org.apache.streams.local.test.processors.PassthroughDatumCounterProcessor;
import org.apache.streams.local.test.processors.SlowProcessor;
@@ -81,10 +82,17 @@ public class LocalStreamBuilderTest extends RandomizedTest {
} catch (MalformedObjectNameException|InstanceNotFoundException|MBeanRegistrationException e) {
//No-op
}
+ try {
+ mbs.unregisterMBean(new ObjectName((String.format(StreamsTaskCounter.NAME_TEMPLATE, id))));
+ } catch (MalformedObjectNameException|InstanceNotFoundException|MBeanRegistrationException e) {
+ //No-op
+ }
}
}
+
+
@Test
public void testStreamIdValidations() {
StreamBuilder builder = new LocalStreamBuilder();
@@ -104,7 +112,7 @@ public class LocalStreamBuilderTest extends RandomizedTest {
exp = e;
}
assertNotNull(exp);
- removeRegisteredMBeans("1", "2");
+ removeRegisteredMBeans("1", "2", "id");
}
@Test
@@ -160,9 +168,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
}
} finally {
for(int i=0; i < numProcessors; ++i) {
- removeRegisteredMBeans(processorId+i);
+ removeRegisteredMBeans(processorId+i, processorId+i+"-"+PassthroughDatumCounterProcessor.class.getCanonicalName());
}
- removeRegisteredMBeans("writer");
+ removeRegisteredMBeans("writer", "numeric_provider");
}
}
@@ -199,7 +207,7 @@ public class LocalStreamBuilderTest extends RandomizedTest {
for(int i=0; i < numProcessors; ++i) {
removeRegisteredMBeans(processorId+i);
}
- removeRegisteredMBeans("writer");
+ removeRegisteredMBeans("writer", "numeric_provider");
}
}
@@ -221,7 +229,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
assertEquals(numDatums2, PassthroughDatumCounterProcessor.COUNTS.get("proc2").get());
assertEquals(numDatums1+numDatums2, DatumCounterWriter.COUNTS.get("writer").get());
} finally {
- removeRegisteredMBeans("proc1", "proc2", "writer1");
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
+ removeRegisteredMBeans("proc1", "proc2", "writer1", "sp1", "sp2");
}
}
@@ -239,6 +249,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
assertEquals(numDatums, PassthroughDatumCounterProcessor.COUNTS.get("proc2").get());
assertEquals(numDatums*2, DatumCounterWriter.COUNTS.get("writer").get());
} finally {
+ String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
removeRegisteredMBeans("prov1", "proc1", "proc2", "w1");
}
}
@@ -257,6 +270,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
builder.start();
assertEquals(numDatums, DatumCounterWriter.COUNTS.get("writer").get());
} finally {
+ String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
removeRegisteredMBeans("prov1", "proc1", "w1");
}
}
@@ -278,6 +294,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
//We care mostly that it doesn't terminate too early. With thread shutdowns, etc, the actual time is indeterminate. Just make sure there is an upper bound
assertThat((int) (end - start), is(allOf(greaterThanOrEqualTo(timeout), lessThanOrEqualTo(4 * timeout))));
} finally {
+ String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
removeRegisteredMBeans("prov1", "proc1", "proc2", "w1");
}
}
@@ -305,6 +324,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
service.awaitTermination(30000, TimeUnit.MILLISECONDS);
assertThat(Thread.activeCount(), is(equalTo(before)));
} finally {
+ String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
removeRegisteredMBeans("prov1", "proc1", "w1");
}
}
[23/38] incubator-streams git commit: STREAMS-203 | Broke down
deserialization method into more reasonably sized chunks
Posted by sb...@apache.org.
STREAMS-203 | Broke down deserialization method into more reasonably sized chunks
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/48b8b71b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/48b8b71b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/48b8b71b
Branch: refs/heads/STREAMS-49
Commit: 48b8b71b69e08eb08688056e1a7fa221d875b018
Parents: c921f67
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Mon Nov 3 09:50:37 2014 -0600
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Mon Nov 3 09:50:37 2014 -0600
----------------------------------------------------------------------
.../util/GPlusActivityDeserializer.java | 147 ++++++++++++-------
1 file changed, 90 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/48b8b71b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
index 9cb1d61..988e138 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
@@ -50,7 +50,7 @@ public class GPlusActivityDeserializer extends JsonDeserializer<Activity> {
* @throws JsonProcessingException
*/
@Override
- public Activity deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+ public Activity deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
JsonNode node = jsonParser.getCodec().readTree(jsonParser);
Activity activity = new Activity();
@@ -64,66 +64,99 @@ public class GPlusActivityDeserializer extends JsonDeserializer<Activity> {
activity.setId(node.get("id").asText());
activity.setVerb(node.get("verb").asText());
- Activity.Actor actor = new Activity.Actor();
- JsonNode actorNode = node.get("actor");
-
- actor.setId(actorNode.get("id").asText());
- actor.setDisplayName(actorNode.get("displayName").asText());
- actor.setUrl(actorNode.get("url").asText());
-
- Activity.Actor.Image image = new Activity.Actor.Image();
- JsonNode imageNode = actorNode.get("image");
- image.setUrl(imageNode.get("url").asText());
-
- actor.setImage(image);
- activity.setActor(actor);
-
- Activity.PlusObject object = new Activity.PlusObject();
- JsonNode objectNode = node.get("object");
- object.setObjectType(objectNode.get("objectType").asText());
- object.setContent(objectNode.get("content").asText());
- object.setUrl(objectNode.get("url").asText());
-
- Activity.PlusObject.Replies replies = new Activity.PlusObject.Replies();
- JsonNode repliesNode = objectNode.get("replies");
- replies.setTotalItems(repliesNode.get("totalItems").asLong());
- replies.setSelfLink(repliesNode.get("selfLink").asText());
- object.setReplies(replies);
-
- Activity.PlusObject.Plusoners plusoners = new Activity.PlusObject.Plusoners();
- JsonNode plusonersNode = objectNode.get("plusoners");
- plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
- plusoners.setSelfLink(plusonersNode.get("selfLink").asText());
- object.setPlusoners(plusoners);
-
- Activity.PlusObject.Resharers resharers = new Activity.PlusObject.Resharers();
- JsonNode resharersNode = objectNode.get("resharers");
- resharers.setTotalItems(resharersNode.get("totalItems").asLong());
- resharers.setSelfLink(resharersNode.get("selfLink").asText());
- object.setResharers(resharers);
-
- List<Activity.PlusObject.Attachments> attachments = Lists.newArrayList();
- for (JsonNode attachmentNode : objectNode.get("attachments")) {
- Activity.PlusObject.Attachments attachments1 = new Activity.PlusObject.Attachments();
- attachments1.setObjectType(attachmentNode.get("objectType").asText());
- attachments1.setDisplayName(attachmentNode.get("displayName").asText());
- attachments1.setContent(attachmentNode.get("content").asText());
- attachments1.setUrl(attachmentNode.get("url").asText());
-
- Activity.PlusObject.Attachments.Image image1 = new Activity.PlusObject.Attachments.Image();
- JsonNode imageNode1 = attachmentNode.get("image");
- image1.setUrl(imageNode1.get("url").asText());
- attachments1.setImage(image1);
-
- attachments.add(attachments1);
- }
- object.setAttachments(attachments);
-
- activity.setObject(object);
+ activity.setActor(buildActor(node));
+
+ activity.setObject(buildPlusObject(node));
} catch (Exception e) {
LOGGER.error("Exception while trying to deserialize activity object: {}", e);
}
return activity;
}
+
+ /**
+ * Given a raw JsonNode, build out the G+ {@link com.google.api.services.plus.model.Activity.Actor} object
+ *
+ * @param node
+ * @return {@link com.google.api.services.plus.model.Activity.Actor} object
+ */
+ private Activity.Actor buildActor(JsonNode node) {
+ Activity.Actor actor = new Activity.Actor();
+ JsonNode actorNode = node.get("actor");
+
+ actor.setId(actorNode.get("id").asText());
+ actor.setDisplayName(actorNode.get("displayName").asText());
+ actor.setUrl(actorNode.get("url").asText());
+
+ Activity.Actor.Image image = new Activity.Actor.Image();
+ JsonNode imageNode = actorNode.get("image");
+ image.setUrl(imageNode.get("url").asText());
+
+ actor.setImage(image);
+
+ return actor;
+ }
+
+ /**
+ * Given a JsonNode, build out all aspects of the {@link com.google.api.services.plus.model.Activity.PlusObject} object
+ *
+ * @param node
+ * @return {@link com.google.api.services.plus.model.Activity.PlusObject} object
+ */
+ private Activity.PlusObject buildPlusObject(JsonNode node) {
+ Activity.PlusObject object = new Activity.PlusObject();
+ JsonNode objectNode = node.get("object");
+ object.setObjectType(objectNode.get("objectType").asText());
+ object.setContent(objectNode.get("content").asText());
+ object.setUrl(objectNode.get("url").asText());
+
+ Activity.PlusObject.Replies replies = new Activity.PlusObject.Replies();
+ JsonNode repliesNode = objectNode.get("replies");
+ replies.setTotalItems(repliesNode.get("totalItems").asLong());
+ replies.setSelfLink(repliesNode.get("selfLink").asText());
+ object.setReplies(replies);
+
+ Activity.PlusObject.Plusoners plusoners = new Activity.PlusObject.Plusoners();
+ JsonNode plusonersNode = objectNode.get("plusoners");
+ plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
+ plusoners.setSelfLink(plusonersNode.get("selfLink").asText());
+ object.setPlusoners(plusoners);
+
+ Activity.PlusObject.Resharers resharers = new Activity.PlusObject.Resharers();
+ JsonNode resharersNode = objectNode.get("resharers");
+ resharers.setTotalItems(resharersNode.get("totalItems").asLong());
+ resharers.setSelfLink(resharersNode.get("selfLink").asText());
+ object.setResharers(resharers);
+
+ object.setAttachments(buildAttachments(objectNode));//attachments);
+
+ return object;
+ }
+
+ /**
+ * Given a raw JsonNode representation of an Activity's attachments, build out that
+ * list of {@link com.google.api.services.plus.model.Activity.PlusObject.Attachments} objects
+ *
+ * @param objectNode
+ * @return list of {@link com.google.api.services.plus.model.Activity.PlusObject.Attachments} objects
+ */
+ private List<Activity.PlusObject.Attachments> buildAttachments(JsonNode objectNode) {
+ List<Activity.PlusObject.Attachments> attachments = Lists.newArrayList();
+ for (JsonNode attachmentNode : objectNode.get("attachments")) {
+ Activity.PlusObject.Attachments attachments1 = new Activity.PlusObject.Attachments();
+ attachments1.setObjectType(attachmentNode.get("objectType").asText());
+ attachments1.setDisplayName(attachmentNode.get("displayName").asText());
+ attachments1.setContent(attachmentNode.get("content").asText());
+ attachments1.setUrl(attachmentNode.get("url").asText());
+
+ Activity.PlusObject.Attachments.Image image1 = new Activity.PlusObject.Attachments.Image();
+ JsonNode imageNode1 = attachmentNode.get("image");
+ image1.setUrl(imageNode1.get("url").asText());
+ attachments1.setImage(image1);
+
+ attachments.add(attachments1);
+ }
+
+ return attachments;
+ }
}
\ No newline at end of file
[32/38] incubator-streams git commit: simple Integration Test
Posted by sb...@apache.org.
simple Integration Test
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/dbd7d890
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/dbd7d890
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/dbd7d890
Branch: refs/heads/STREAMS-49
Commit: dbd7d890804391025cf085f8a1c141a643d0b94b
Parents: b8e7a69
Author: sblackmon <sb...@apache.org>
Authored: Thu Nov 6 09:45:32 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Fri Nov 7 10:47:21 2014 -0800
----------------------------------------------------------------------
pom.xml | 56 ++++++++++++++++++++
streams-contrib/streams-persist-console/pom.xml | 34 ++++++++++++
.../streams/console/ConsolePersistReader.java | 11 ++--
.../streams/console/ConsolePersistWriter.java | 11 ++--
4 files changed, 103 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/dbd7d890/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f8b6a14..e86f02d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,6 +74,8 @@
<jaxb2-basics.version>0.8.4</jaxb2-basics.version>
<jaxbutil.version>1.2.6</jaxbutil.version>
<junit.version>4.11</junit.version>
+ <surefire.plugin.version>2.17</surefire.plugin.version>
+ <failsafe.plugin.version>2.17</failsafe.plugin.version>
<slf4j.version>1.7.6</slf4j.version>
<logback.version>1.1.1</logback.version>
<commons-io.version>2.4</commons-io.version>
@@ -90,6 +92,7 @@
<facebook4j.version>2.1.0</facebook4j.version>
<maven.enforcer.plugin.version>1.3.1</maven.enforcer.plugin.version>
<mockito.version>1.9.5</mockito.version>
+ <powermock.version>1.5.6</powermock.version>
</properties>
<modules>
@@ -177,6 +180,41 @@
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper.version}</version>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${failsafe.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <!-- Sets the VM argument line used when integration tests are run. -->
+ <argLine>${failsafeArgLine}</argLine>
+ <!-- Skips integration tests if the value of skip.integration.tests property is true -->
+ <skipTests>${skip.integration.tests}</skipTests>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.plugin.version}</version>
+ <configuration>
+ <!-- Sets the VM argument line used when unit tests are run. -->
+ <argLine>${surefireArgLine}</argLine>
+ <!-- Skips unit tests if the value of skip.unit.tests property is true -->
+ <skipTests>${skip.unit.tests}</skipTests>
+ <!-- Excludes integration tests when unit tests are run. -->
+ <excludes>
+ <exclude>**/IT*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
</build>
@@ -231,6 +269,24 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/dbd7d890/streams-contrib/streams-persist-console/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/pom.xml b/streams-contrib/streams-persist-console/pom.xml
index c7f2cd3..02ec403 100644
--- a/streams-contrib/streams-persist-console/pom.xml
+++ b/streams-contrib/streams-persist-console/pom.xml
@@ -26,5 +26,39 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-runtime-local</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ </dependency>
</dependencies>
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ </build>
+
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/dbd7d890/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
index 776d5a3..8afba85 100644
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
+++ b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
@@ -31,6 +31,7 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.InputStream;
+import java.io.PrintStream;
import java.math.BigInteger;
import java.util.Queue;
import java.util.Scanner;
@@ -44,16 +45,16 @@ public class ConsolePersistReader implements StreamsPersistReader {
protected volatile Queue<StreamsDatum> persistQueue;
- private ObjectMapper mapper = new ObjectMapper();
+ protected InputStream inputStream = System.in;
public ConsolePersistReader() {
this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
}
- public ConsolePersistReader(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
+ public ConsolePersistReader(InputStream inputStream) {
+ this();
+ this.inputStream = inputStream;
}
-
public void prepare(Object o) {
}
@@ -77,7 +78,7 @@ public class ConsolePersistReader implements StreamsPersistReader {
LOGGER.info("{} readCurrent", STREAMS_ID);
- Scanner sc = new Scanner(System.in);
+ Scanner sc = new Scanner(inputStream);
while( sc.hasNextLine() ) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/dbd7d890/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
index 96d116f..53bb8d7 100644
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
+++ b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
@@ -27,6 +27,7 @@ import org.apache.streams.jackson.StreamsJacksonMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.PrintStream;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -34,6 +35,8 @@ public class ConsolePersistWriter implements StreamsPersistWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(ConsolePersistWriter.class);
+ protected PrintStream printStream = System.out;
+
protected volatile Queue<StreamsDatum> persistQueue;
private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
@@ -42,8 +45,9 @@ public class ConsolePersistWriter implements StreamsPersistWriter {
this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
}
- public ConsolePersistWriter(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
+ public ConsolePersistWriter(PrintStream printStream) {
+ this();
+ this.printStream = printStream;
}
public void prepare(Object o) {
@@ -61,8 +65,7 @@ public class ConsolePersistWriter implements StreamsPersistWriter {
String text = mapper.writeValueAsString(entry);
- System.out.println("\n"+text+"\n");
-// LOGGER.info(text);
+ printStream.println(text);
} catch (JsonProcessingException e) {
LOGGER.warn("save: {}", e);
[14/38] incubator-streams git commit: Merge branch
'depricated_monitor' into countableImpl
Posted by sb...@apache.org.
Merge branch 'depricated_monitor' into countableImpl
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/015fadea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/015fadea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/015fadea
Branch: refs/heads/STREAMS-49
Commit: 015fadeac499777185f38056f2308272c8404794
Parents: d71568d 4f2e3cf
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Mon Oct 20 16:18:00 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Mon Oct 20 16:18:00 2014 -0500
----------------------------------------------------------------------
.../org/apache/streams/core/DatumStatusCountable.java | 1 +
.../java/org/apache/streams/core/DatumStatusCounter.java | 5 +++++
.../apache/streams/local/builders/StreamComponent.java | 1 +
.../streams/local/counters/StreamsTaskCounter.java | 11 ++++++++---
.../local/tasks/LocalStreamProcessMonitorThread.java | 1 +
.../local/tasks/StatusCounterMonitorRunnable.java | 1 +
.../streams/local/tasks/StatusCounterMonitorThread.java | 1 +
7 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[08/38] incubator-streams git commit: committing PeoplePattern
processor
Posted by sb...@apache.org.
committing PeoplePattern processor
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/9a575322
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/9a575322
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/9a575322
Branch: refs/heads/STREAMS-49
Commit: 9a575322231e4a8a69c56f06da743ffe3211ccb4
Parents: d62061d
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Oct 12 20:20:52 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Oct 12 20:20:52 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 5 +-
.../streams-processor-peoplepattern/pom.xml | 138 +++++++++++++++++++
.../peoplepattern/AccountTypeProcessor.java | 76 ++++++++++
.../peoplepattern/DemographicsProcessor.java | 77 +++++++++++
.../streams/peoplepattern/AccountType.json | 27 ++++
.../streams/peoplepattern/Demographics.json | 60 ++++++++
.../resources/templates/peoplepatternactor.json | 25 ++++
7 files changed, 406 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
index e290466..fcec297 100644
--- a/streams-contrib/pom.xml
+++ b/streams-contrib/pom.xml
@@ -44,17 +44,18 @@
<module>streams-persist-hdfs</module>
<module>streams-persist-kafka</module>
<module>streams-persist-mongo</module>
- <module>streams-amazon-aws</module>
+ <module>streams-amazon-aws</module>
<!--<module>streams-processor-lucene</module>-->
<!--<module>streams-processor-tika</module>-->
- <module>streams-provider-instagram</module>
<module>streams-processor-jackson</module>
<module>streams-processor-json</module>
<module>streams-processor-urls</module>
+ <module>streams-processor-peoplepattern</module>
<module>streams-provider-datasift</module>
<module>streams-provider-facebook</module>
<module>streams-provider-google</module>
<module>streams-provider-gnip</module>
+ <module>streams-provider-instagram</module>
<module>streams-provider-moreover</module>
<module>streams-provider-twitter</module>
<module>streams-provider-sysomos</module>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/pom.xml b/streams-contrib/streams-processor-peoplepattern/pom.xml
new file mode 100644
index 0000000..b810200
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/pom.xml
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>streams-processor-peoplepattern</artifactId>
+ <version>0.1-SNAPSHOT</version>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-contrib</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-http</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo/**/*.java</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-source-jaxb2</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jaxb2</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema/org/apache/streams/peoplepattern</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>org.apache.streams.peoplepattern</targetPackage>
+ <useLongIntegers>true</useLongIntegers>
+ <useJodaDates>true</useJodaDates>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
new file mode 100644
index 0000000..d180b7f
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.peoplepattern;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Maps;
+import org.apache.streams.components.http.HttpConfigurator;
+import org.apache.streams.components.http.HttpProcessorConfiguration;
+import org.apache.streams.components.http.processor.SimpleHTTPGetProcessor;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.data.util.ExtensionUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * Enrich actor with demographics
+ */
+public class AccountTypeProcessor extends SimpleHTTPGetProcessor {
+
+ private final static String STREAMS_ID = "AccountTypeProcessor";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(AccountTypeProcessor.class);
+
+ public AccountTypeProcessor() {
+ this(HttpConfigurator.detectProcessorConfiguration(StreamsConfigurator.config.getConfig("peoplepattern")));
+ }
+
+ public AccountTypeProcessor(HttpProcessorConfiguration peoplePatternConfiguration) {
+ super(peoplePatternConfiguration);
+ LOGGER.info("creating AccountTypeProcessor");
+ configuration.setProtocol("https");
+ configuration.setHostname("api.peoplepattern.com");
+ configuration.setResourcePath("/v0.2/account_type/");
+ configuration.setEntity(HttpProcessorConfiguration.Entity.ACTOR);
+ configuration.setExtension("account_type");
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ @Override
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+ Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
+ //Actor actor = mapper.convertValue(entry.getDocument(), Actor.class);
+ Actor actor = activity.getActor();
+ ObjectNode actorObjectNode = mapper.convertValue(actor, ObjectNode.class);
+ String username = (String) ExtensionUtil.getExtension(actorObjectNode, "screenName");
+ Map<String, String> params = Maps.newHashMap();
+ params.put("id", actor.getId());
+ params.put("name", actor.getDisplayName());
+ params.put("username", username);
+ params.put("description", actor.getSummary());
+ return params;
+ }
+};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
new file mode 100644
index 0000000..6ffbb9b
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.peoplepattern;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Maps;
+import org.apache.streams.components.http.HttpConfigurator;
+import org.apache.streams.components.http.HttpProcessorConfiguration;
+import org.apache.streams.components.http.processor.SimpleHTTPGetProcessor;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.data.util.ExtensionUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * Enrich actor with demographics
+ */
+public class DemographicsProcessor extends SimpleHTTPGetProcessor {
+
+ public final static String STREAMS_ID = "DemographicsProcessor";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(DemographicsProcessor.class);
+
+ public DemographicsProcessor() {
+ this(HttpConfigurator.detectProcessorConfiguration(StreamsConfigurator.config.getConfig("peoplepattern")));
+ }
+
+ public DemographicsProcessor(HttpProcessorConfiguration peoplePatternConfiguration) {
+ super(peoplePatternConfiguration);
+ LOGGER.info("creating DemographicsProcessor");
+ configuration.setProtocol("https");
+ configuration.setHostname("api.peoplepattern.com");
+ configuration.setResourcePath("/v0.2/demographics/");
+ configuration.setEntity(HttpProcessorConfiguration.Entity.ACTOR);
+ configuration.setExtension("demographics");
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ @Override
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+ Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
+ //Actor actor = mapper.convertValue(entry.getDocument(), Actor.class);
+ Actor actor = activity.getActor();
+ ObjectNode actorObjectNode = mapper.convertValue(actor, ObjectNode.class);
+ String username = (String) ExtensionUtil.getExtension(actorObjectNode, "screenName");
+ Map<String, String> params = Maps.newHashMap();
+ params.put("id", actor.getId());
+ params.put("name", actor.getDisplayName());
+ params.put("username", username);
+ params.put("description", actor.getSummary());
+ return params;
+ }
+
+};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json
new file mode 100644
index 0000000..5656b44
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json
@@ -0,0 +1,27 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType": "org.apache.streams.peoplepattern.AccountType",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "prediction" : {
+ "type" : "string",
+ "enum" : [
+ "person",
+ "organization",
+ "entertainment",
+ "adult",
+ "spam",
+ "no-prediction"
+ ],
+ "default": "no-prediction"
+ },
+ "score": {
+ "type": "number"
+ },
+ "id": {
+ "type": "string"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json
new file mode 100644
index 0000000..d1f64d8
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json
@@ -0,0 +1,60 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType": "org.apache.streams.peoplepattern.Demographics",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "age": {
+ "type": "object",
+ "properties": {
+ "prediction": {
+ "type": "integer",
+ "default": 1990
+ },
+ "score": {
+ "type": "number"
+ }
+
+ }
+ },
+ "gender" : {
+ "type": "object",
+ "properties": {
+ "prediction": {
+ "type": "string",
+ "enum": [
+ "male",
+ "female",
+ "no-prediction"
+ ],
+ "default": "no-prediction"
+ },
+ "score": {
+ "type": "number"
+ }
+ }
+ },
+ "race" : {
+ "type": "object",
+ "properties": {
+ "prediction": {
+ "type": "string",
+ "enum": [
+ "black",
+ "east-asian",
+ "hispanic",
+ "middle-eastern",
+ "south-asian",
+ "white",
+ "no-prediction"
+ ],
+ "default": "no-prediction"
+ },
+ "score": {
+ "type": "number"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json b/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json
new file mode 100644
index 0000000..9a24c5c
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json
@@ -0,0 +1,25 @@
+{
+ "order": 20,
+ "template": "*activity*",
+ "settings": {},
+ "mappings": {
+ "activity": {
+ "properties": {
+ "actor": {
+ "properties": {
+ "extensions": {
+ "properties": {
+ "account_type": {
+ "type": "nested"
+ },
+ "demographics": {
+ "type": "nested"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
[10/38] incubator-streams git commit: a few package name changes
working twitterurlapiprocess which uses streams-http (w/o authentication)
working people pattern processors which use streams-http (w/ authentication)
Posted by sb...@apache.org.
a few package name changes
working twitterurlapiprocess which uses streams-http (w/o authentication)
working people pattern processors which use streams-http (w/ authentication)
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/0c8e67ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/0c8e67ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/0c8e67ce
Branch: refs/heads/STREAMS-49
Commit: 0c8e67ce26e131282aaaa03815ad6daefd684346
Parents: ad5f90c
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Oct 12 23:13:29 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Oct 12 23:13:29 2014 -0500
----------------------------------------------------------------------
.../http/processor/SimpleHTTPGetProcessor.java | 50 +++++++++++++++-----
.../peoplepattern/AccountTypeProcessor.java | 7 ++-
.../peoplepattern/DemographicsProcessor.java | 7 ++-
.../streams-provider-twitter/pom.xml | 2 +-
.../processor/TwitterUrlApiProcessor.java | 21 ++++----
.../provider/TwitterTimelineProvider.java | 8 ++--
.../apache/streams/data/util/ExtensionUtil.java | 37 ++++++---------
7 files changed, 77 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
index 0d17cc6..d3d4429 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
@@ -21,6 +21,7 @@ import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.data.util.ExtensionUtil;
import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.ActivityObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -99,15 +100,30 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
}
}
- /**
- Override this to place result in non-standard location on document
- */
- protected ObjectNode getEntityToExtend(ObjectNode rootDocument) {
+
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ActivityObject getEntityToExtend(ObjectNode rootDocument) {
+
+ if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
+ return mapper.convertValue(rootDocument, ActivityObject.class);
+ else
+ return mapper.convertValue(rootDocument.get(this.configuration.getEntity().toString()), ActivityObject.class);
+
+ }
+
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode setEntityToExtend(ObjectNode rootDocument, ActivityObject activityObject) {
if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
- return rootDocument;
+ return mapper.convertValue(activityObject, ObjectNode.class);
else
- return (ObjectNode) rootDocument.get(this.configuration.getEntity().toString());
+ rootDocument.set(this.configuration.getEntity().toString(), mapper.convertValue(activityObject, ObjectNode.class));
+
+ return rootDocument;
}
@@ -150,9 +166,6 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
try {
response.close();
} catch (IOException e) {}
- try {
- httpclient.close();
- } catch (IOException e) {}
}
if( entityString == null )
@@ -162,12 +175,12 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
ObjectNode extensionFragment = prepareExtensionFragment(entityString);
- ObjectNode extensionEntity = getEntityToExtend(rootDocument);
-
- ExtensionUtil.ensureExtensions(extensionEntity);
+ ActivityObject extensionEntity = getEntityToExtend(rootDocument);
ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
+ rootDocument = setEntityToExtend(rootDocument, extensionEntity);
+
entry.setDocument(rootDocument);
result.add(entry);
@@ -220,5 +233,18 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
@Override
public void cleanUp() {
LOGGER.info("shutting down SimpleHTTPGetProcessor");
+ try {
+ httpclient.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ httpclient.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ httpclient = null;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
index d180b7f..edcf4d3 100644
--- a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
@@ -18,7 +18,6 @@
package org.apache.streams.peoplepattern;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Maps;
import org.apache.streams.components.http.HttpConfigurator;
import org.apache.streams.components.http.HttpProcessorConfiguration;
@@ -27,6 +26,7 @@ import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.data.util.ExtensionUtil;
import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
import org.apache.streams.pojo.json.Actor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,10 +62,9 @@ public class AccountTypeProcessor extends SimpleHTTPGetProcessor {
@Override
protected Map<String, String> prepareParams(StreamsDatum entry) {
Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
- //Actor actor = mapper.convertValue(entry.getDocument(), Actor.class);
Actor actor = activity.getActor();
- ObjectNode actorObjectNode = mapper.convertValue(actor, ObjectNode.class);
- String username = (String) ExtensionUtil.getExtension(actorObjectNode, "screenName");
+ ActivityObject actorObject = mapper.convertValue(actor, ActivityObject.class);
+ String username = (String) ExtensionUtil.getExtension(actorObject, "screenName");
Map<String, String> params = Maps.newHashMap();
params.put("id", actor.getId());
params.put("name", actor.getDisplayName());
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
index 6ffbb9b..60db379 100644
--- a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
@@ -18,7 +18,6 @@
package org.apache.streams.peoplepattern;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Maps;
import org.apache.streams.components.http.HttpConfigurator;
import org.apache.streams.components.http.HttpProcessorConfiguration;
@@ -27,6 +26,7 @@ import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.data.util.ExtensionUtil;
import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
import org.apache.streams.pojo.json.Actor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,10 +62,9 @@ public class DemographicsProcessor extends SimpleHTTPGetProcessor {
@Override
protected Map<String, String> prepareParams(StreamsDatum entry) {
Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
- //Actor actor = mapper.convertValue(entry.getDocument(), Actor.class);
Actor actor = activity.getActor();
- ObjectNode actorObjectNode = mapper.convertValue(actor, ObjectNode.class);
- String username = (String) ExtensionUtil.getExtension(actorObjectNode, "screenName");
+ ActivityObject actorObject = mapper.convertValue(actor, ActivityObject.class);
+ String username = (String) ExtensionUtil.getExtension(actorObject, "screenName");
Map<String, String> params = Maps.newHashMap();
params.put("id", actor.getId());
params.put("name", actor.getDisplayName());
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
index f0d65f8..604e5a7 100644
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@ -55,7 +55,7 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-processor-http</artifactId>
+ <artifactId>streams-http</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
index 438937f..54e1369 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
@@ -1,11 +1,10 @@
package org.apache.streams.twitter.processor;
import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.streams.components.http.HttpProcessorConfiguration;
-import org.apache.streams.components.http.SimpleHTTPGetProcessor;
-import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.components.http.processor.SimpleHTTPGetProcessor;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.pojo.json.Activity;
@@ -21,23 +20,27 @@ public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements St
public TwitterUrlApiProcessor() {
super();
this.configuration.setHostname("urls.api.twitter.com");
- this.configuration.setResourceUri("/1/urls/count.json");
+ this.configuration.setResourcePath("/1/urls/count.json");
+ this.configuration.setEntity(HttpProcessorConfiguration.Entity.ACTIVITY);
this.configuration.setExtension("twitter_url_count");
}
public TwitterUrlApiProcessor(HttpProcessorConfiguration processorConfiguration) {
super(processorConfiguration);
this.configuration.setHostname("urls.api.twitter.com");
- this.configuration.setResourceUri("/1/urls/count.json");
+ this.configuration.setResourcePath("/1/urls/count.json");
+ this.configuration.setEntity(HttpProcessorConfiguration.Entity.ACTIVITY);
this.configuration.setExtension("twitter_url_count");
}
@Override
public List<StreamsDatum> process(StreamsDatum entry) {
Preconditions.checkArgument(entry.getDocument() instanceof Activity);
- Activity activity = mapper.convertValue(entry, Activity.class);
- Preconditions.checkArgument(!Strings.isNullOrEmpty(activity.getUrl()));
- return super.process(entry);
+ Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
+ if( activity.getLinks() != null && activity.getLinks().size() > 0)
+ return super.process(entry);
+ else
+ return Lists.newArrayList(entry);
}
@Override
@@ -45,7 +48,7 @@ public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements St
Map<String, String> params = Maps.newHashMap();
- params.put("url", mapper.convertValue(entry, Activity.class).getUrl());
+ params.put("url", mapper.convertValue(entry.getDocument(), Activity.class).getLinks().get(0));
return params;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index ae755c2..86395a2 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -19,9 +19,9 @@
package org.apache.streams.twitter.provider;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Queues;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.streams.core.DatumStatusCounter;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProvider;
@@ -31,14 +31,16 @@ import org.apache.streams.util.ComponentUtils;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
import twitter4j.*;
import twitter4j.conf.ConfigurationBuilder;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.*;
-import java.util.concurrent.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
index a8d068a..7ce013c 100644
--- a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
+++ b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
@@ -1,7 +1,6 @@
package org.apache.streams.data.util;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Maps;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.ActivityObject;
@@ -41,38 +40,33 @@ public class ExtensionUtil {
private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- public static Map<String, Object> getExtensions(ObjectNode object) {
+ public static Map<String, Object> getExtensions(ActivityObject object) {
ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ Map<String,Object> extensions = ensureExtensions(object);
return extensions;
}
- public static Object getExtension(ObjectNode object, String key) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ public static Object getExtension(ActivityObject object, String key) {
+ Map<String,Object> extensions = ensureExtensions(object);
return extensions.get(key);
}
- public static void setExtensions(ObjectNode object, Map<String, Object> extensions) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- activityObject.setAdditionalProperty(EXTENSION_PROPERTY, extensions);
+ public static void setExtensions(ActivityObject object, Map<String, Object> extensions) {
+ object.setAdditionalProperty(EXTENSION_PROPERTY, extensions);
};
- public static void addExtension(ObjectNode object, String key, Object extension) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ public static void addExtension(ActivityObject object, String key, Object extension) {
+ Map<String,Object> extensions = ensureExtensions(object);
extensions.put(key, extension);
};
- public static void addExtensions(ObjectNode object, Map<String, Object> extensions) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ public static void addExtensions(ActivityObject object, Map<String, Object> extensions) {
for( Map.Entry<String, Object> item : extensions.entrySet())
- activityObject.getAdditionalProperties().put(item.getKey(), item.getValue());
+ addExtension(object, item.getKey(), item.getValue());
};
- public static void removeExtension(ObjectNode object, String key) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ public static void removeExtension(ActivityObject object, String key) {
+ Map<String,Object> extensions = ensureExtensions(object);
extensions.remove(key);
};
@@ -82,13 +76,12 @@ public class ExtensionUtil {
* @return the Map representing the extensions property
*/
@SuppressWarnings("unchecked")
- public static Map<String, Object> ensureExtensions(ObjectNode object) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ public static Map<String, Object> ensureExtensions(ActivityObject object) {
+ Map<String,Object> extensions = (Map<String,Object>) object.getAdditionalProperties().get(EXTENSION_PROPERTY);
if(extensions == null) {
extensions = Maps.newHashMap();
setExtensions(object, extensions);
}
- return getExtensions(object);
+ return extensions;
}
}
[11/38] incubator-streams git commit: added license header and
javadoc comments
Posted by sb...@apache.org.
added license header and javadoc comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d2ffe62b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d2ffe62b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d2ffe62b
Branch: refs/heads/STREAMS-49
Commit: d2ffe62baf88618c7c8a6b9a1e5a79d21ed9e3fd
Parents: 0c8e67c
Author: sblackmon <sb...@apache.org>
Authored: Mon Oct 13 17:07:19 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Mon Oct 13 17:07:19 2014 -0500
----------------------------------------------------------------------
.../components/http/HttpConfigurator.java | 2 +-
.../http/processor/SimpleHTTPGetProcessor.java | 18 +++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 18 +++++++++++++++++
.../peoplepattern/AccountTypeProcessor.java | 2 +-
.../processor/TwitterUrlApiProcessor.java | 20 ++++++++++++++++++-
.../apache/streams/data/util/ExtensionUtil.java | 21 ++++++++++++++++++++
6 files changed, 78 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
index 900831f..979a680 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
@@ -25,7 +25,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
+ * Converts a {@link com.typesafe.config.Config} element into an instance of HttpConfiguration
*/
public class HttpConfigurator {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
index d3d4429..b8c957c 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.components.http.processor;
import com.fasterxml.jackson.core.JsonProcessingException;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
index 36084c7..118d06b 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.components.http.provider;
import com.fasterxml.jackson.databind.JsonNode;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
index edcf4d3..4e4a6af 100644
--- a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
import java.util.Map;
/**
- * Enrich actor with demographics
+ * Enrich actor with account type
*/
public class AccountTypeProcessor extends SimpleHTTPGetProcessor {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
index 54e1369..17ce411 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.twitter.processor;
import com.google.common.base.Preconditions;
@@ -13,7 +31,7 @@ import java.util.List;
import java.util.Map;
/**
- * Created by sblackmon on 9/14/14.
+ * Class gets a global share count from Twitter API for links on Activity datums
*/
public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements StreamsProcessor {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
index 7ce013c..1e0e384 100644
--- a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
+++ b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.data.util;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -7,6 +25,9 @@ import org.apache.streams.pojo.json.ActivityObject;
import java.util.Map;
+/**
+ * Class makes it easier to manage extensions added to activities, actors, objects, etc...
+ */
public class ExtensionUtil {
/**
[24/38] incubator-streams git commit: Merge branch 'STREAMS-206'
Posted by sb...@apache.org.
Merge branch 'STREAMS-206'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/3341415d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/3341415d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/3341415d
Branch: refs/heads/STREAMS-49
Commit: 3341415d9811f915bf931fc406ed3039798b4530
Parents: d88e8c8 1ee614e
Author: sblackmon <sb...@apache.org>
Authored: Mon Nov 3 10:16:09 2014 -0600
Committer: sblackmon <sb...@apache.org>
Committed: Mon Nov 3 10:16:09 2014 -0600
----------------------------------------------------------------------
.../elasticsearch/processor/PercolateTagProcessor.java | 10 ++++++----
.../elasticsearch/ElasticsearchWriterConfiguration.json | 10 ++++++++++
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[27/38] incubator-streams git commit: STREAMS-204 | Initial commit
Posted by sb...@apache.org.
STREAMS-204 | Initial commit
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/2358b4e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/2358b4e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/2358b4e7
Branch: refs/heads/STREAMS-49
Commit: 2358b4e76d3b0f47d07eb569ef6affefbea0dee0
Parents: 9aebd0b
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Wed Oct 29 17:42:11 2014 -0500
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Thu Nov 6 14:04:13 2014 -0600
----------------------------------------------------------------------
.../google/gplus/processor/GooglePlusCommentProcessor.java | 7 +++++++
.../gplus/serializer/util/GPlusCommentDeserializer.java | 7 +++++++
.../java/com/google/gplus/GooglePLusCommentSerDeTest.java | 7 +++++++
.../src/test/resources/google_plus_comments_jsons.txt | 0
4 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2358b4e7/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
new file mode 100644
index 0000000..7dc4d56
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
@@ -0,0 +1,7 @@
+package com.google.gplus.processor;
+
+/**
+ * Created by rdouglas on 10/29/14.
+ */
+public class GooglePlusCommentProcessor {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2358b4e7/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
new file mode 100644
index 0000000..51446e0
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
@@ -0,0 +1,7 @@
+package com.google.gplus.serializer.util;
+
+/**
+ * Created by rdouglas on 10/29/14.
+ */
+public class GPlusCommentDeserializer {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2358b4e7/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
new file mode 100644
index 0000000..b3479f9
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
@@ -0,0 +1,7 @@
+package com.google.gplus;
+
+/**
+ * Created by rdouglas on 10/29/14.
+ */
+public class GooglePLusCommentSerDeTest {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2358b4e7/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
new file mode 100644
index 0000000..e69de29
[03/38] incubator-streams git commit: added HttpConfigurator
Posted by sb...@apache.org.
added HttpConfigurator
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/b8ccf9f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/b8ccf9f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/b8ccf9f6
Branch: refs/heads/STREAMS-49
Commit: b8ccf9f6869c64f1f6e25f668942ea0de9fec2eb
Parents: 34232ad
Author: sblackmon <sb...@apache.org>
Authored: Mon Sep 15 18:38:09 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Mon Sep 15 18:38:09 2014 -0500
----------------------------------------------------------------------
.../components/http/HttpConfigurator.java | 53 ++++++++++++++++++++
.../components/http/SimpleHTTPGetProcessor.java | 23 +++++----
.../processor/TwitterUrlApiProcessor.java | 8 +++
3 files changed, 75 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8ccf9f6/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
new file mode 100644
index 0000000..36801b8
--- /dev/null
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.components.http;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigRenderOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
+ */
+public class HttpConfigurator {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(HttpConfigurator.class);
+
+ private final static ObjectMapper mapper = new ObjectMapper();
+
+ public static HttpProcessorConfiguration detectConfiguration(Config config) {
+
+ HttpProcessorConfiguration httpProcessorConfiguration = null;
+
+ try {
+ httpProcessorConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), HttpProcessorConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse http configuration", e.getMessage());
+ }
+ return httpProcessorConfiguration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8ccf9f6/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
index d76d839..dec9d03 100644
--- a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
@@ -19,6 +19,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.data.util.ActivityUtil;
@@ -34,6 +35,7 @@ import javax.validation.ValidatorFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -59,6 +61,11 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
// //private String authHeader;
//
+ public SimpleHTTPGetProcessor() {
+ LOGGER.info("creating SimpleHTTPGetProcessor");
+ this.configuration = HttpConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("http"));
+ }
+
public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
LOGGER.info("creating SimpleHTTPGetProcessor");
LOGGER.info(processorConfiguration.toString());
@@ -137,9 +144,7 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
return result;
}
- HttpGet httpget = new HttpGet(uri);
- httpget.addHeader("content-type", this.configuration.getContentType());
- //httpget.addHeader("Authorization", String.format("Basic %s", authHeader));
+ HttpGet httpget = prepareHttpGet(uri);
CloseableHttpResponse response = null;
@@ -184,6 +189,12 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
}
+ public HttpGet prepareHttpGet(URI uri) {
+ HttpGet httpget = new HttpGet(uri);
+ httpget.addHeader("content-type", this.configuration.getContentType());
+ return httpget;
+ }
+
@Override
public void prepare(Object configurationObject) {
@@ -198,12 +209,6 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
.setPath(this.configuration.getResourceUri());
httpclient = HttpClients.createDefault();
- // StringBuilder stringBuilder = new StringBuilder();
-// stringBuilder.append(peoplePatternConfiguration.getUsername());
-// stringBuilder.append(":");
-// stringBuilder.append(peoplePatternConfiguration.getPassword());
-// String string = stringBuilder.toString();
-// authHeader = Base64.encodeBase64String(string.getBytes());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8ccf9f6/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
index 77965c4..438937f 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
@@ -5,6 +5,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import org.apache.streams.components.http.HttpProcessorConfiguration;
import org.apache.streams.components.http.SimpleHTTPGetProcessor;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.pojo.json.Activity;
@@ -17,6 +18,13 @@ import java.util.Map;
*/
public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements StreamsProcessor {
+ public TwitterUrlApiProcessor() {
+ super();
+ this.configuration.setHostname("urls.api.twitter.com");
+ this.configuration.setResourceUri("/1/urls/count.json");
+ this.configuration.setExtension("twitter_url_count");
+ }
+
public TwitterUrlApiProcessor(HttpProcessorConfiguration processorConfiguration) {
super(processorConfiguration);
this.configuration.setHostname("urls.api.twitter.com");
[30/38] incubator-streams git commit: STREAMS-204 | Added test back
in with correct name
Posted by sb...@apache.org.
STREAMS-204 | Added test back in with correct name
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/b8e7a69d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/b8e7a69d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/b8e7a69d
Branch: refs/heads/STREAMS-49
Commit: b8e7a69d08edfa615c96722da30bf01f0b2d9b8b
Parents: e792bb9
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Thu Nov 6 14:11:00 2014 -0600
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Thu Nov 6 14:12:16 2014 -0600
----------------------------------------------------------------------
.../gplus/GooglePlusCommentSerDeTest.java | 114 +++++++++++++++++++
1 file changed, 114 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8e7a69d/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeTest.java
new file mode 100644
index 0000000..9fea22c
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.google.gplus;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Comment;
+import com.google.gplus.serializer.util.GPlusCommentDeserializer;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class GooglePlusCommentSerDeTest {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentSerDeTest.class);
+ private ObjectMapper objectMapper;
+ private GooglePlusActivityUtil googlePlusActivityUtil;
+
+ @Before
+ public void setup() {
+ objectMapper = new StreamsJacksonMapper();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Comment.class, new GPlusCommentDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ googlePlusActivityUtil = new GooglePlusActivityUtil();
+ }
+
+ @org.junit.Test
+ public void testCommentObjects() {
+ InputStream is = GooglePlusPersonSerDeTest.class.getResourceAsStream("/google_plus_comments_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ Activity activity = new Activity();
+ List<Comment> comments = Lists.newArrayList();
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Comment comment = objectMapper.readValue(line, Comment.class);
+
+ LOGGER.info("comment: {}", comment);
+
+ assertNotNull(comment);
+ assertNotNull(comment.getEtag());
+ assertNotNull(comment.getId());
+ assertNotNull(comment.getInReplyTo());
+ assertNotNull(comment.getObject());
+ assertNotNull(comment.getPlusoners());
+ assertNotNull(comment.getPublished());
+ assertNotNull(comment.getUpdated());
+ assertNotNull(comment.getSelfLink());
+ assertEquals(comment.getVerb(), "post");
+
+ comments.add(comment);
+ }
+ }
+
+ assertEquals(comments.size(), 3);
+
+ googlePlusActivityUtil.updateActivity(comments, activity);
+ assertNotNull(activity);
+ assertNotNull(activity.getObject());
+ assertEquals(activity.getObject().getAttachments().size(), 3);
+ } catch (Exception e) {
+ LOGGER.error("Exception while testing serializability: {}", e);
+ }
+ }
+
+ @org.junit.Test
+ public void testEmptyComments() {
+ Activity activity = new Activity();
+
+ googlePlusActivityUtil.updateActivity(new ArrayList<Comment>(), activity);
+
+ assertNull(activity.getObject());
+ }
+}
[20/38] incubator-streams git commit: Merge pull request #11 from
apache/master
Posted by sb...@apache.org.
Merge pull request #11 from apache/master
Merge Apache master 2014/10/31
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/5c2e8322
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/5c2e8322
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/5c2e8322
Branch: refs/heads/STREAMS-49
Commit: 5c2e8322fb407df1d4059609a205569f71bc2b31
Parents: 2c71550 d88e8c8
Author: Ryan Ebanks <rb...@users.noreply.github.com>
Authored: Fri Oct 31 16:22:28 2014 -0500
Committer: Ryan Ebanks <rb...@users.noreply.github.com>
Committed: Fri Oct 31 16:22:28 2014 -0500
----------------------------------------------------------------------
streams-contrib/streams-amazon-aws/pom.xml | 2 +-
.../org/apache/streams/s3/S3Configurator.java | 55 +++----
.../org/apache/streams/s3/S3PersistReader.java | 11 +-
.../org/apache/streams/s3/S3PersistWriter.java | 10 +-
.../org/apache/streams/s3/S3Configuration.json | 18 ++-
.../google-gplus/pom.xml | 2 +-
.../processor/GooglePlusTypeConverter.java | 91 +++++++++++
.../util/GPlusPersonDeserializer.java | 107 +++++++++++++
.../serializer/util/GooglePlusActivityUtil.java | 115 ++++++++++++++
.../google/gplus/GooglePlusPersonSerDeTest.java | 97 ++++++++++++
.../processor/GooglePlusTypeConverterTest.java | 98 ++++++++++++
.../test/resources/google_plus_person_jsons.txt | 2 +
.../streams/core/DatumStatusCountable.java | 2 +
.../apache/streams/core/DatumStatusCounter.java | 7 +-
streams-runtimes/streams-runtime-local/pom.xml | 6 +-
.../streams/local/builders/StreamComponent.java | 1 +
.../local/counters/DatumStatusCounter.java | 83 ++++++++++
.../counters/DatumStatusCounterMXBean.java | 43 +++++
.../local/counters/StreamsTaskCounter.java | 157 ++++++++++++++++++
.../counters/StreamsTaskCounterMXBean.java | 63 ++++++++
...amOnUnhandleThrowableThreadPoolExecutor.java | 17 ++
.../tasks/LocalStreamProcessMonitorThread.java | 1 +
.../tasks/StatusCounterMonitorRunnable.java | 1 +
.../local/tasks/StatusCounterMonitorThread.java | 1 +
.../local/counters/DatumStatusCounterTest.java | 134 ++++++++++++++++
.../local/counters/StreamsTaskCounterTest.java | 158 +++++++++++++++++++
.../org/apache/streams/util/ComponentUtils.java | 19 ++-
27 files changed, 1253 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
[09/38] incubator-streams git commit: removing validation,
as jackson does not contain a default implementation until 2.5 release
Posted by sb...@apache.org.
removing validation, as jackson does not contain a default implementation until 2.5 release
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ad5f90cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ad5f90cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ad5f90cc
Branch: refs/heads/STREAMS-49
Commit: ad5f90cc115c40e0ef480a85957bbb4c98157cf6
Parents: 9a57532
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Oct 12 20:21:22 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Oct 12 20:21:22 2014 -0500
----------------------------------------------------------------------
streams-components/streams-http/pom.xml | 6 ------
.../components/http/processor/SimpleHTTPGetProcessor.java | 6 ------
.../components/http/provider/SimpleHTTPGetProvider.java | 3 ---
3 files changed, 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ad5f90cc/streams-components/streams-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/pom.xml b/streams-components/streams-http/pom.xml
index 9c2b079..39a4faa 100644
--- a/streams-components/streams-http/pom.xml
+++ b/streams-components/streams-http/pom.xml
@@ -38,12 +38,6 @@
<artifactId>jsonschema2pojo-core</artifactId>
<type>jar</type>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ad5f90cc/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
index c2bfef6..0d17cc6 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
@@ -3,7 +3,6 @@ package org.apache.streams.components.http.processor;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -25,8 +24,6 @@ import org.apache.streams.jackson.StreamsJacksonMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.validation.Validation;
-import javax.validation.ValidatorFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -199,9 +196,6 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
@Override
public void prepare(Object configurationObject) {
- ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
- Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
-
mapper = StreamsJacksonMapper.getInstance();
uriBuilder = new URIBuilder()
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ad5f90cc/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
index 622225a..36084c7 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
@@ -89,9 +89,6 @@ public class SimpleHTTPGetProvider implements StreamsProvider {
@Override
public void prepare(Object configurationObject) {
-// ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
-// Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
-
mapper = StreamsJacksonMapper.getInstance();
uriBuilder = new URIBuilder()
[12/38] incubator-streams git commit: Implemented new
StreamTasksCounter into StreamsTasks
Posted by sb...@apache.org.
Implemented new StreamTasksCounter into StreamsTasks
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/7e65a423
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/7e65a423
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/7e65a423
Branch: refs/heads/STREAMS-49
Commit: 7e65a423f31c2fc4540b72ade6be904c77638784
Parents: d305371
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Mon Oct 20 14:16:40 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Mon Oct 20 14:16:40 2014 -0500
----------------------------------------------------------------------
.../streams/local/tasks/StreamsMergeTask.java | 7 ++++
.../local/tasks/StreamsPersistWriterTask.java | 18 +++++++--
.../local/tasks/StreamsProcessorTask.java | 20 ++++++++--
.../local/tasks/StreamsProviderTask.java | 15 +++++++
.../apache/streams/local/tasks/StreamsTask.java | 4 ++
.../streams/local/tasks/BasicTasksTest.java | 41 ++++++++++++++++++--
6 files changed, 93 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
index 7a4c806..8280f29 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
@@ -19,6 +19,8 @@
package org.apache.streams.local.tasks;
import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.local.counters.StreamsTaskCounter;
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -78,4 +80,9 @@ public class StreamsMergeTask extends BaseStreamsTask {
}
}
}
+
+ @Override
+ public void setStreamsTaskCounter(StreamsTaskCounter counter) {
+ throw new NotImplementedException();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
index cab46b8..003ab9e 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
@@ -20,13 +20,11 @@ package org.apache.streams.local.tasks;
import org.apache.streams.core.*;
import org.apache.streams.core.util.DatumUtils;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
+import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -45,6 +43,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
private BlockingQueue<StreamsDatum> inQueue;
private AtomicBoolean isRunning;
private AtomicBoolean blocked;
+ private StreamsTaskCounter counter;
private DatumStatusCounter statusCounter = new DatumStatusCounter();
@@ -99,6 +98,9 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
public void run() {
try {
this.writer.prepare(this.streamConfig);
+ if(this.counter == null) {
+ this.counter = new StreamsTaskCounter(this.writer.getClass().getName()+ UUID.randomUUID().toString());
+ }
while(this.keepRunning.get()) {
StreamsDatum datum = null;
try {
@@ -111,14 +113,18 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
Thread.currentThread().interrupt();
}
if(datum != null) {
+ this.counter.incrementReceivedCount();
try {
+ long startTime = System.currentTimeMillis();
this.writer.write(datum);
+ this.counter.addTime(System.currentTimeMillis() - startTime);
statusCounter.incrementStatus(DatumStatus.SUCCESS);
} catch (Exception e) {
LOGGER.error("Error writing to persist writer {}", this.writer.getClass().getSimpleName(), e);
this.keepRunning.set(false); // why do we shutdown on a failed write ?
statusCounter.incrementStatus(DatumStatus.FAIL);
DatumUtils.addErrorToMetadata(datum, e, this.writer.getClass());
+ this.counter.incrementErrorCount();
}
} else { //datums should never be null
LOGGER.debug("Received null StreamsDatum @ writer : {}", this.writer.getClass().getName());
@@ -151,4 +157,8 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
return queues;
}
+ @Override
+ public void setStreamsTaskCounter(StreamsTaskCounter counter) {
+ this.counter = counter;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index ee69127..8d66847 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -21,13 +21,11 @@ package org.apache.streams.local.tasks;
import com.google.common.collect.Maps;
import org.apache.streams.core.*;
import org.apache.streams.core.util.DatumUtils;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
+import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -47,6 +45,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
private BlockingQueue<StreamsDatum> inQueue;
private AtomicBoolean isRunning;
private AtomicBoolean blocked;
+ private StreamsTaskCounter counter;
private DatumStatusCounter statusCounter = new DatumStatusCounter();
@@ -105,6 +104,9 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
public void run() {
try {
this.processor.prepare(this.streamConfig);
+ if(this.counter == null) {
+ this.counter = new StreamsTaskCounter(this.processor.getClass().getName()+ UUID.randomUUID().toString());
+ }
while(this.keepRunning.get()) {
StreamsDatum datum = null;
try {
@@ -117,11 +119,15 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
Thread.currentThread().interrupt();
}
if(datum != null) {
+ this.counter.incrementReceivedCount();
try {
+ long startTime = System.currentTimeMillis();
List<StreamsDatum> output = this.processor.process(datum);
+ this.counter.addTime(System.currentTimeMillis() - startTime);
if(output != null) {
for(StreamsDatum outDatum : output) {
super.addToOutgoingQueue(datum);
+ this.counter.incrementEmittedCount();
statusCounter.incrementStatus(DatumStatus.SUCCESS);
}
}
@@ -130,6 +136,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
this.keepRunning.set(false);
Thread.currentThread().interrupt();
} catch (Throwable t) {
+ this.counter.incrementErrorCount();
LOGGER.warn("Caught Throwable in processor, {} : {}", this.processor.getClass().getName(), t.getMessage());
statusCounter.incrementStatus(DatumStatus.FAIL);
//Add the error to the metadata, but keep processing
@@ -151,4 +158,9 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
queues.add(this.inQueue);
return queues;
}
+
+ @Override
+ public void setStreamsTaskCounter(StreamsTaskCounter counter) {
+ this.counter = counter;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index c16f64d..2475780 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -20,6 +20,7 @@ package org.apache.streams.local.tasks;
import org.apache.streams.core.*;
import org.apache.streams.core.util.DatumUtils;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,6 +28,7 @@ import org.slf4j.LoggerFactory;
import java.math.BigInteger;
import java.util.Map;
import java.util.Queue;
+import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -64,6 +66,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
private long sleepTime;
private int zeros = 0;
private DatumStatusCounter statusCounter = new DatumStatusCounter();
+ private StreamsTaskCounter counter;
/**
* Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readCurrent()}
@@ -145,13 +148,18 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
StreamsResultSet resultSet = null;
//Negative values mean we want to run forever
long maxZeros = timeout < 0 ? Long.MAX_VALUE : (timeout / sleepTime);
+ if(this.counter == null) { //should never be null
+ this.counter = new StreamsTaskCounter(this.provider.getClass().getName()+ UUID.randomUUID().toString());
+ }
switch(this.type) {
case PERPETUAL: {
provider.startStream();
this.started.set(true);
while(this.isRunning()) {
try {
+ long startTime = System.currentTimeMillis();
resultSet = provider.readCurrent();
+ this.counter.addTime(System.currentTimeMillis() - startTime);
if( resultSet.size() == 0 )
zeros++;
else {
@@ -164,6 +172,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
if(zeros > 0)
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
+ this.counter.incrementErrorCount();
LOGGER.warn("Thread interrupted");
this.keepRunning.set(false);
}
@@ -219,8 +228,10 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
if(datum != null) {
try {
super.addToOutgoingQueue(datum);
+ this.counter.incrementEmittedCount();
statusCounter.incrementStatus(DatumStatus.SUCCESS);
} catch( Exception e ) {
+ this.counter.incrementErrorCount();
statusCounter.incrementStatus(DatumStatus.FAIL);
DatumUtils.addErrorToMetadata(datum, e, this.provider.getClass());
}
@@ -229,4 +240,8 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
this.flushing.set(false);
}
+ @Override
+ public void setStreamsTaskCounter(StreamsTaskCounter counter) {
+ this.counter = counter;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
index 7513631..8423095 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
@@ -19,6 +19,7 @@
package org.apache.streams.local.tasks;
import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import java.util.List;
import java.util.Map;
@@ -87,4 +88,7 @@ public interface StreamsTask extends Runnable{
*/
public List<BlockingQueue<StreamsDatum>> getOutputQueues();
+
+ public void setStreamsTaskCounter(StreamsTaskCounter counter);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
index f524db0..f62250d 100644
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
@@ -19,12 +19,18 @@
package org.apache.streams.local.tasks;
import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.local.counters.DatumStatusCounter;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.streams.local.queues.ThroughputQueue;
import org.apache.streams.local.test.processors.PassthroughDatumCounterProcessor;
import org.apache.streams.local.test.providers.NumericMessageProvider;
import org.apache.streams.local.test.writer.DatumCounterWriter;
+import org.junit.After;
import org.junit.Test;
+import javax.management.InstanceNotFoundException;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
import java.util.Queue;
import java.util.concurrent.*;
@@ -36,6 +42,21 @@ import static org.junit.Assert.*;
public class BasicTasksTest {
+ private static final String MBEAN_ID = "test_bean";
+
+ /**
+ * Remove registered mbeans from previous tests
+ * @throws Exception
+ */
+ @After
+ public void unregisterMXBean() throws Exception {
+ try {
+ ManagementFactory.getPlatformMBeanServer().unregisterMBean(new ObjectName(String.format(StreamsTaskCounter.NAME_TEMPLATE, MBEAN_ID)));
+ } catch (InstanceNotFoundException ife) {
+ //No-op
+ }
+ }
+
@Test
public void testProviderTask() {
@@ -77,7 +98,7 @@ public class BasicTasksTest {
assertTrue("Task should have completed running in aloted time.", service.isTerminated());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
- };
+ }
}
@Test
@@ -85,6 +106,8 @@ public class BasicTasksTest {
int numMessages = 100;
PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor("");
StreamsProcessorTask task = new StreamsProcessorTask(processor);
+ StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID);
+ task.setStreamsTaskCounter(counter);
BlockingQueue<StreamsDatum> outQueue = new LinkedBlockingQueue<>();
BlockingQueue<StreamsDatum> inQueue = createInputQueue(numMessages);
task.addOutputQueue(outQueue);
@@ -104,8 +127,7 @@ public class BasicTasksTest {
fail("Processor task failed to output "+numMessages+" in a timely fashion.");
}
}
- task.stopTask();
- assertEquals(numMessages, processor.getMessageCount());
+ task.stopTask();;
service.shutdown();
try {
if(!service.awaitTermination(5, TimeUnit.SECONDS)){
@@ -116,6 +138,11 @@ public class BasicTasksTest {
} catch (InterruptedException e) {
fail("Test Interupted.");
}
+ assertEquals(numMessages, processor.getMessageCount());
+ assertEquals(numMessages, counter.getNumReceived());
+ assertEquals(numMessages, counter.getNumEmitted());
+ assertEquals(0, counter.getNumUnhandledErrors());
+ assertEquals(0.0, counter.getErrorRate(), 0.0);
}
@Test
@@ -123,6 +150,8 @@ public class BasicTasksTest {
int numMessages = 100;
DatumCounterWriter writer = new DatumCounterWriter("");
StreamsPersistWriterTask task = new StreamsPersistWriterTask(writer);
+ StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID);
+ task.setStreamsTaskCounter(counter);
BlockingQueue<StreamsDatum> outQueue = new LinkedBlockingQueue<>();
BlockingQueue<StreamsDatum> inQueue = createInputQueue(numMessages);
@@ -150,7 +179,6 @@ public class BasicTasksTest {
}
}
task.stopTask();
- assertEquals(numMessages, writer.getDatumsCounted());
service.shutdown();
try {
if(!service.awaitTermination(5, TimeUnit.SECONDS)){
@@ -161,6 +189,11 @@ public class BasicTasksTest {
} catch (InterruptedException e) {
fail("Test Interupted.");
}
+ assertEquals(numMessages, writer.getDatumsCounted());
+ assertEquals(numMessages, counter.getNumReceived());
+ assertEquals(0, counter.getNumEmitted());
+ assertEquals(0, counter.getNumUnhandledErrors());
+ assertEquals(0.0, counter.getErrorRate(), 0.0);
}
@Test
[31/38] incubator-streams git commit: kafka IT
Posted by sb...@apache.org.
kafka IT
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/409b4482
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/409b4482
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/409b4482
Branch: refs/heads/STREAMS-49
Commit: 409b44827e047a2bd2c07effb5f4127e04a4b7da
Parents: 85e0b50
Author: sblackmon <sb...@apache.org>
Authored: Thu Nov 6 13:28:37 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Thu Nov 6 13:28:37 2014 -0800
----------------------------------------------------------------------
streams-contrib/streams-persist-kafka/pom.xml | 24 ++++
.../apache/streams/kafka/KafkaConfigurator.java | 20 ++--
.../streams/kafka/KafkaPersistReaderTask.java | 22 ++--
.../streams/kafka/KafkaPersistWriter.java | 15 +--
.../streams/kafka/StreamsPartitioner.java | 2 +-
.../streams/kafka/test/KafkaPersistIT.java | 109 +++++++++++++++++++
6 files changed, 165 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/409b4482/streams-contrib/streams-persist-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-kafka/pom.xml b/streams-contrib/streams-persist-kafka/pom.xml
index 4e9e375..6d98c36 100644
--- a/streams-contrib/streams-persist-kafka/pom.xml
+++ b/streams-contrib/streams-persist-kafka/pom.xml
@@ -85,6 +85,30 @@
<version>2.6.0</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-runtime-local</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-persist-console</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/409b4482/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaConfigurator.java b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaConfigurator.java
index 9f64ae6..4a835e8 100644
--- a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaConfigurator.java
+++ b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaConfigurator.java
@@ -18,7 +18,9 @@
package org.apache.streams.kafka;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.typesafe.config.Config;
+import com.typesafe.config.ConfigRenderOptions;
import org.apache.streams.config.StreamsConfigurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,18 +32,18 @@ public class KafkaConfigurator {
private final static Logger LOGGER = LoggerFactory.getLogger(KafkaConfigurator.class);
+ private final static ObjectMapper mapper = new ObjectMapper();
+
public static KafkaConfiguration detectConfiguration(Config kafka) {
- String brokerlist = StreamsConfigurator.config.getString("kafka.metadata.broker.list");
- String zkconnect = StreamsConfigurator.config.getString("kafka.zkconnect");
- String topic = StreamsConfigurator.config.getString("kafka.topic");
- String groupId = StreamsConfigurator.config.getString("kafka.groupid");
- KafkaConfiguration kafkaConfiguration = new KafkaConfiguration();
+ KafkaConfiguration kafkaConfiguration = null;
- kafkaConfiguration.setBrokerlist(brokerlist);
- kafkaConfiguration.setZkconnect(zkconnect);
- kafkaConfiguration.setTopic(topic);
- kafkaConfiguration.setGroupId(groupId);
+ try {
+ kafkaConfiguration = mapper.readValue(kafka.root().render(ConfigRenderOptions.concise()), KafkaConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse KafkaConfiguration");
+ }
return kafkaConfiguration;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/409b4482/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReaderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReaderTask.java b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReaderTask.java
index 03fa291..3c71398 100644
--- a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReaderTask.java
+++ b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReaderTask.java
@@ -33,30 +33,38 @@ public class KafkaPersistReaderTask implements Runnable {
private static final Logger LOGGER = LoggerFactory.getLogger(KafkaPersistReaderTask.class);
private KafkaPersistReader reader;
- private KafkaStream<String,String> stream;
+ private KafkaStream<String,Object> stream;
- public KafkaPersistReaderTask(KafkaPersistReader reader, KafkaStream<String,String> stream) {
+ public KafkaPersistReaderTask(KafkaPersistReader reader, KafkaStream<String,Object> stream) {
this.reader = reader;
this.stream = stream;
}
-
-
@Override
public void run() {
Preconditions.checkNotNull(this.stream);
- ConsumerIterator<String, String> it = stream.iterator();
+ ConsumerIterator<String, Object> it = stream.iterator();
while (it.hasNext()) {
- MessageAndMetadata<String,String> item = it.next();
- reader.persistQueue.add(new StreamsDatum(item.message()));
+ MessageAndMetadata<String,Object> item = it.next();
+ write(new StreamsDatum(item.message()));
}
try {
Thread.sleep(new Random().nextInt(100));
} catch (InterruptedException e) {}
+ }
+ private void write( StreamsDatum entry ) {
+ boolean success;
+ do {
+ synchronized( KafkaPersistReader.class ) {
+ success = reader.persistQueue.offer(entry);
+ }
+ Thread.yield();
+ }
+ while( !success );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/409b4482/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java
index 01db32f..2937b7a 100644
--- a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java
+++ b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java
@@ -46,7 +46,7 @@ public class KafkaPersistWriter implements StreamsPersistWriter, Serializable {
private KafkaConfiguration config;
- private Producer<String, String> producer;
+ private Producer<String, Object> producer;
public KafkaPersistWriter() {
this(KafkaConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("kafka")));
@@ -67,7 +67,7 @@ public class KafkaPersistWriter implements StreamsPersistWriter, Serializable {
ProducerConfig config = new ProducerConfig(props);
- producer = new Producer<String, String>(config);
+ producer = new Producer<String, Object>(config);
new Thread(new KafkaPersistWriterTask(this)).start();
}
@@ -75,16 +75,11 @@ public class KafkaPersistWriter implements StreamsPersistWriter, Serializable {
@Override
public void write(StreamsDatum entry) {
- try {
- String text = mapper.writeValueAsString(entry);
+ String key = entry.getId() != null ? entry.getId() : GuidUtils.generateGuid("kafkawriter");
- KeyedMessage<String, String> data = new KeyedMessage<String, String>(config.getTopic(), entry.getId(), text);
+ KeyedMessage<String, Object> data = new KeyedMessage<>(config.getTopic(), key, entry.getDocument());
- producer.send(data);
-
- } catch (JsonProcessingException e) {
- LOGGER.warn("save: {}", e);
- }// put
+ producer.send(data);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/409b4482/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/StreamsPartitioner.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/StreamsPartitioner.java b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/StreamsPartitioner.java
index fa38ca2..b1f2a28 100644
--- a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/StreamsPartitioner.java
+++ b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/StreamsPartitioner.java
@@ -32,7 +32,7 @@ public class StreamsPartitioner implements Partitioner {
public int partition(Object key, int a_numPartitions) {
int partition = 0;
- partition = key.hashCode() % a_numPartitions;
+
return partition;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/409b4482/streams-contrib/streams-persist-kafka/src/test/java/org/apache/streams/kafka/test/KafkaPersistIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-kafka/src/test/java/org/apache/streams/kafka/test/KafkaPersistIT.java b/streams-contrib/streams-persist-kafka/src/test/java/org/apache/streams/kafka/test/KafkaPersistIT.java
new file mode 100644
index 0000000..2817fca
--- /dev/null
+++ b/streams-contrib/streams-persist-kafka/src/test/java/org/apache/streams/kafka/test/KafkaPersistIT.java
@@ -0,0 +1,109 @@
+package org.apache.streams.kafka.test;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Queues;
+import javassist.bytecode.stackmap.TypeData;
+import kafka.admin.AdminUtils;
+import kafka.utils.ZKStringSerializer$;
+import org.I0Itec.zkclient.ZkClient;
+import org.apache.streams.console.ConsolePersistReader;
+import org.apache.streams.console.ConsolePersistWriter;
+import org.apache.streams.core.StreamBuilder;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.kafka.KafkaConfiguration;
+import org.apache.streams.kafka.KafkaPersistReader;
+import org.apache.streams.kafka.KafkaPersistWriter;
+import org.apache.streams.local.builders.LocalStreamBuilder;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
+import org.powermock.api.mockito.PowerMockito;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * Created by sblackmon on 10/20/14.
+ */
+public class KafkaPersistIT {
+
+ private TestKafkaCluster testKafkaCluster;
+ private KafkaConfiguration testConfiguration;
+
+ private String testTopic = "testTopic";
+
+ ConsolePersistReader reader = Mockito.mock(ConsolePersistReader.class);
+ ConsolePersistWriter writer = Mockito.mock(ConsolePersistWriter.class);
+
+ StreamsDatum testDatum = new StreamsDatum("{\"dummy\":\"true\"}");
+
+ @Before
+ public void prepareTest() {
+
+ try {
+ testKafkaCluster = new TestKafkaCluster();
+ } catch (Throwable e ) {
+ e.printStackTrace();
+ }
+
+ testConfiguration = new KafkaConfiguration();
+ testConfiguration.setBrokerlist(testKafkaCluster.getKafkaBrokerString());
+ testConfiguration.setZkconnect(testKafkaCluster.getZkConnectString());
+ testConfiguration.setTopic(testTopic);
+
+ ZkClient zkClient = new ZkClient(testKafkaCluster.getZkConnectString(), 1000, 1000, ZKStringSerializer$.MODULE$);
+
+ AdminUtils.createTopic(zkClient, testTopic, 1, 1, new Properties());
+
+ PowerMockito.when(reader.readCurrent())
+ .thenReturn(
+ new StreamsResultSet(Queues.newConcurrentLinkedQueue(
+ Lists.newArrayList(testDatum)))
+ );
+ }
+
+ @Test
+ public void testPersistStream() {
+
+ assert(testConfiguration != null);
+ assert(testKafkaCluster != null);
+
+ Map<String, Object> streamConfig = Maps.newHashMap();
+ streamConfig.put(LocalStreamBuilder.TIMEOUT_KEY, 1000);
+
+ StreamBuilder builder = new LocalStreamBuilder(1, streamConfig);
+
+ KafkaPersistWriter kafkaWriter = new KafkaPersistWriter(testConfiguration);
+ KafkaPersistReader kafkaReader = new KafkaPersistReader(testConfiguration);
+
+ builder.newReadCurrentStream("stdin", reader);
+ builder.addStreamsPersistWriter("writer", kafkaWriter, 1, "stdin");
+ builder.newPerpetualStream("reader", kafkaReader);
+ builder.addStreamsPersistWriter("stdout", writer, 1, "reader");
+
+ builder.start();
+
+ builder.stop();
+
+ Mockito.verify(writer).write(testDatum);
+
+ }
+}
[15/38] incubator-streams git commit: STREAMS-202 | Built a
GooglePlusTypeConverter,
the corresponding deserializer and all necessary tests
Posted by sb...@apache.org.
STREAMS-202 | Built a GooglePlusTypeConverter, the corresponding deserializer and all necessary tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/eba9aefd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/eba9aefd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/eba9aefd
Branch: refs/heads/STREAMS-49
Commit: eba9aefd77273c8290324ac33370aa250da34c10
Parents: eec3aa9
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Tue Oct 28 12:02:51 2014 -0500
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Tue Oct 28 12:02:51 2014 -0500
----------------------------------------------------------------------
.../processor/GooglePlusTypeConverter.java | 7 +
.../util/GPlusPersonDeserializer.java | 7 +
.../serializer/util/GooglePlusActivityUtil.java | 7 +
.../google/gplus/GooglePlusPersonSerDeTest.java | 139 +++++++++++++++++++
.../processor/GooglePlusTypeConverterTest.java | 7 +
.../test/resources/google_plus_person_jsons.txt | 0
6 files changed, 167 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/eba9aefd/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
new file mode 100644
index 0000000..f23ca17
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
@@ -0,0 +1,7 @@
+package com.google.gplus.processor;
+
+/**
+ * Created by rdouglas on 10/27/14.
+ */
+public class GooglePlusTypeConverter {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/eba9aefd/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
new file mode 100644
index 0000000..48fed99
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
@@ -0,0 +1,7 @@
+package com.google.gplus.serializer.util;
+
+/**
+ * Created by rdouglas on 10/28/14.
+ */
+public class GPlusPersonDeserializer {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/eba9aefd/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
new file mode 100644
index 0000000..a3150c8
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
@@ -0,0 +1,7 @@
+package com.google.gplus.serializer.util;
+
+/**
+ * Created by rdouglas on 10/27/14.
+ */
+public class GooglePlusActivityUtil {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/eba9aefd/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeTest.java
new file mode 100644
index 0000000..22cc340
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.google.gplus;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.*;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.node.IntNode;
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Person;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.apache.streams.pojo.json.Provider;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class GooglePlusPersonSerDeTest {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusPersonSerDeTest.class);
+
+ @Test
+ public void TestPersonObjects() {
+ ObjectMapper objectMapper = new StreamsJacksonMapper();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
+ objectMapper.registerModule(simpleModule);
+
+ InputStream is = GooglePlusPersonSerDeTest.class.getResourceAsStream("/google_plus_person_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ GooglePlusActivityUtil googlePlusActivityUtil = new GooglePlusActivityUtil();
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ Activity activity = new Activity();
+
+ Person person = objectMapper.readValue(line, Person.class);
+
+ googlePlusActivityUtil.updateActivity(person, activity);
+ LOGGER.info("activity: {}", activity);
+
+ assertNotNull(activity);
+ assert(activity.getId().contains("id:googleplus:update"));
+ assertEquals(activity.getVerb(), "update");
+
+ Provider provider = activity.getProvider();
+ assertEquals(provider.getId(), "id:providers:googleplus");
+ assertEquals(provider.getDisplayName(), "GooglePlus");
+
+ Actor actor = activity.getActor();
+ assertNotNull(actor.getImage());
+ assert(actor.getId().contains("id:googleplus:"));
+ assertNotNull(actor.getUrl());
+
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.error("Exception while testing serializability: {}", e);
+ }
+ }
+}
+
+class GPlusPersonDeserializer extends JsonDeserializer<Person> {
+ @Override
+ public Person deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+ ObjectMapper m = new StreamsJacksonMapper();
+
+ JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+ Person person = new Person();
+
+ person.setCircledByCount((Integer)((IntNode)node.get("circledByCount")).numberValue());
+ person.setDisplayName(node.get("displayName").asText());
+ person.setEtag(node.get("etag").asText());
+ person.setGender(node.get("gender").asText());
+ person.setId(node.get("id").asText());
+
+ Person.Image image = new Person.Image();
+ JsonNode imageNode = node.get("image");
+ image.setIsDefault(imageNode.get("isDefault").asBoolean());
+ image.setUrl(imageNode.get("url").asText());
+ person.setImage(image);
+
+ person.setIsPlusUser(node.get("isPlusUser").asBoolean());
+ person.setKind(node.get("kind").asText());
+
+ JsonNode nameNode = node.get("name");
+ Person.Name name = m.readValue(m.writeValueAsString(nameNode), Person.Name.class);
+ person.setName(name);
+
+ person.setObjectType(node.get("objectType").asText());
+
+ List<Person.Organizations> organizations = Lists.newArrayList();
+ for(JsonNode orgNode : node.get("organizations")) {
+ Person.Organizations org = m.readValue(m.writeValueAsString(orgNode), Person.Organizations.class);
+ organizations.add(org);
+ }
+ person.setOrganizations(organizations);
+
+ person.setUrl(node.get("url").asText());
+ person.setVerified(node.get("verified").asBoolean());
+
+ return person;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/eba9aefd/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
new file mode 100644
index 0000000..e4923ae
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusTypeConverterTest.java
@@ -0,0 +1,7 @@
+package com.google.gplus.processor;
+
+/**
+ * Created by rdouglas on 10/28/14.
+ */
+public class GooglePlusTypeConverterTest {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/eba9aefd/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_person_jsons.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_person_jsons.txt b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_person_jsons.txt
new file mode 100644
index 0000000..e69de29
[38/38] incubator-streams git commit: Merge commit
'01716fe31d6e7507b5a1aaec7e7060de0371c68b' into STREAMS-49
Posted by sb...@apache.org.
Merge commit '01716fe31d6e7507b5a1aaec7e7060de0371c68b' into STREAMS-49
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/3026b831
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/3026b831
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/3026b831
Branch: refs/heads/STREAMS-49
Commit: 3026b83190a8564ba55b23e1130ac7e986a23488
Parents: 073b083 01716fe
Author: sblackmon <sb...@apache.org>
Authored: Fri Nov 7 12:05:07 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Fri Nov 7 12:05:07 2014 -0800
----------------------------------------------------------------------
pom.xml | 2 +
streams-components/pom.xml | 62 +++++
streams-components/streams-http/README.md | 16 ++
streams-components/streams-http/pom.xml | 153 +++++++++++
.../components/http/HttpConfigurator.java | 62 +++++
.../http/processor/SimpleHTTPGetProcessor.java | 268 +++++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 230 ++++++++++++++++
.../components/http/HttpConfiguration.json | 50 ++++
.../http/HttpProcessorConfiguration.json | 28 ++
.../http/HttpProviderConfiguration.json | 18 ++
streams-contrib/pom.xml | 5 +-
.../streams-processor-peoplepattern/pom.xml | 138 ++++++++++
.../peoplepattern/AccountTypeProcessor.java | 75 ++++++
.../peoplepattern/DemographicsProcessor.java | 76 ++++++
.../streams/peoplepattern/AccountType.json | 27 ++
.../streams/peoplepattern/Demographics.json | 60 +++++
.../resources/templates/peoplepatternactor.json | 25 ++
.../api/FacebookPostActivitySerializer.java | 1 -
.../processor/GooglePlusCommentProcessor.java | 87 ++++++
.../util/GPlusCommentDeserializer.java | 98 +++++++
.../serializer/util/GooglePlusActivityUtil.java | 50 ++++
.../gplus/GooglePlusCommentSerDeTest.java | 114 ++++++++
.../resources/google_plus_comments_jsons.txt | 3 +
.../streams-provider-twitter/pom.xml | 5 +
.../processor/TwitterUrlApiProcessor.java | 73 +++++
.../provider/TwitterTimelineProvider.java | 7 +-
streams-pojo-extensions/pom.xml | 64 +++++
.../apache/streams/data/util/ExtensionUtil.java | 108 ++++++++
.../apache/streams/data/util/ActivityUtil.java | 14 +-
.../jackson/StreamsDateTimeDeserializer.java | 23 +-
.../streams/jackson/StreamsJacksonMapper.java | 20 ++
.../streams/jackson/StreamsJacksonModule.java | 9 +
.../org/apache/streams/pojo/json/activity.json | 3 +-
.../org/apache/streams/pojo/json/object.json | 2 +-
.../local/builders/LocalStreamBuilder.java | 5 +
.../streams/local/tasks/StreamsMergeTask.java | 7 +
.../local/tasks/StreamsPersistWriterTask.java | 18 +-
.../local/tasks/StreamsProcessorTask.java | 20 +-
.../local/tasks/StreamsProviderTask.java | 15 ++
.../apache/streams/local/tasks/StreamsTask.java | 4 +
.../local/builders/LocalStreamBuilderTest.java | 32 ++-
.../streams/local/tasks/BasicTasksTest.java | 26 +-
42 files changed, 2072 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
[25/38] incubator-streams git commit: Merge remote-tracking branch
'robdouglas/STREAMS-203'
Posted by sb...@apache.org.
Merge remote-tracking branch 'robdouglas/STREAMS-203'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/9aebd0b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/9aebd0b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/9aebd0b1
Branch: refs/heads/STREAMS-49
Commit: 9aebd0b1a49dccdf31365ac2f0d3043ffd45c762
Parents: 3341415 48b8b71
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Mon Nov 3 10:25:10 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Mon Nov 3 10:25:10 2014 -0600
----------------------------------------------------------------------
.../processor/GooglePlusTypeConverter.java | 5 +-
.../gplus/provider/GPlusActivitySerializer.java | 77 +--------
.../gplus/provider/GPlusEventProcessor.java | 3 -
.../util/GPlusActivityDeserializer.java | 162 +++++++++++++++++++
.../serializer/util/GooglePlusActivityUtil.java | 143 +++++++++++++++-
.../processor/GooglePlusActivitySerDeTest.java | 112 +++++++++++++
.../processor/GooglePlusTypeConverterTest.java | 63 +++++---
.../resources/google_plus_activity_jsons.txt | 5 +
8 files changed, 467 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
[21/38] incubator-streams git commit: Merged apache master
Posted by sb...@apache.org.
Merged apache master
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/377fb897
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/377fb897
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/377fb897
Branch: refs/heads/STREAMS-49
Commit: 377fb89717f34e6e109cf2655d6e3e335a494499
Parents: 015fade 5c2e832
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Oct 31 16:32:16 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Oct 31 16:32:16 2014 -0500
----------------------------------------------------------------------
streams-contrib/streams-amazon-aws/pom.xml | 2 +-
.../org/apache/streams/s3/S3Configurator.java | 55 ++++-----
.../org/apache/streams/s3/S3PersistReader.java | 11 +-
.../org/apache/streams/s3/S3PersistWriter.java | 10 +-
.../org/apache/streams/s3/S3Configuration.json | 18 ++-
.../streams-persist-elasticsearch/pom.xml | 47 +++++++
.../ElasticsearchMetadataUtil.java | 106 ++++++++++++++++
.../ElasticsearchPersistDeleter.java | 6 +-
.../ElasticsearchPersistUpdater.java | 6 +-
.../ElasticsearchPersistWriter.java | 47 +------
.../DatumFromMetadataAsDocumentProcessor.java | 122 +++++++++++++++++++
.../processor/DatumFromMetadataProcessor.java | 103 ++++++++++++++++
.../processor/DocumentToMetadataProcessor.java | 100 +++++++++++++++
.../ElasticsearchConfiguration.json | 3 +-
.../test/TestDatumFromMetadataProcessor.java | 81 ++++++++++++
.../test/TestDocumentToMetadataProcessor.java | 63 ++++++++++
.../test/TestElasticsearchPersistWriter.java | 70 +++++++++++
.../google-gplus/pom.xml | 2 +-
.../processor/GooglePlusTypeConverter.java | 91 ++++++++++++++
.../util/GPlusPersonDeserializer.java | 107 ++++++++++++++++
.../serializer/util/GooglePlusActivityUtil.java | 115 +++++++++++++++++
.../google/gplus/GooglePlusPersonSerDeTest.java | 97 +++++++++++++++
.../processor/GooglePlusTypeConverterTest.java | 98 +++++++++++++++
.../test/resources/google_plus_person_jsons.txt | 2 +
.../provider/TwitterTimelineProvider.java | 14 ++-
.../provider/TwitterTimelineProviderTest.java | 39 ++++++
streams-runtimes/streams-runtime-local/pom.xml | 6 +-
.../local/counters/DatumStatusCounter.java | 15 +--
.../local/counters/StreamsTaskCounter.java | 10 +-
.../local/tasks/StreamsProcessorTask.java | 3 +-
.../local/builders/LocalStreamBuilderTest.java | 12 ++
...nhandledThrowableThreadPoolExecutorTest.java | 11 ++
.../queues/ThroughputQueueMulitThreadTest.java | 10 ++
.../queues/ThroughputQueueSingleThreadTest.java | 11 ++
.../streams/local/tasks/BasicTasksTest.java | 15 +--
.../local/tasks/StreamsProviderTaskTest.java | 10 ++
.../tests/TestComponentsLocalStream.java | 13 +-
.../tests/TestExpectedDatumsPersitWriter.java | 13 +-
.../component/tests/TestFileReaderProvider.java | 12 +-
.../org/apache/streams/util/ComponentUtils.java | 33 +++++
40 files changed, 1455 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/377fb897/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --cc streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index 8d66847,1bb565d..33c5827
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@@ -119,15 -117,11 +119,14 @@@ public class StreamsProcessorTask exten
Thread.currentThread().interrupt();
}
if(datum != null) {
+ this.counter.incrementReceivedCount();
try {
+ long startTime = System.currentTimeMillis();
List<StreamsDatum> output = this.processor.process(datum);
+ this.counter.addTime(System.currentTimeMillis() - startTime);
if(output != null) {
for(StreamsDatum outDatum : output) {
- super.addToOutgoingQueue(datum);
- this.counter.incrementEmittedCount();
+ super.addToOutgoingQueue(outDatum);
statusCounter.incrementStatus(DatumStatus.SUCCESS);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/377fb897/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/377fb897/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
----------------------------------------------------------------------
diff --cc streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
index f62250d,2d28602..a0e28cd
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
@@@ -42,18 -38,12 +43,13 @@@ import static org.junit.Assert.*
public class BasicTasksTest {
+ private static final String MBEAN_ID = "test_bean";
-
- /**
- * Remove registered mbeans from previous tests
- * @throws Exception
- */
@After
- public void unregisterMXBean() throws Exception {
+ public void removeLocalMBeans() {
try {
- ManagementFactory.getPlatformMBeanServer().unregisterMBean(new ObjectName(String.format(StreamsTaskCounter.NAME_TEMPLATE, MBEAN_ID)));
- } catch (InstanceNotFoundException ife) {
- //No-op
+ ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
+ } catch (Exception e) {
+ //No op. proceed to next test
}
}
[34/38] incubator-streams git commit: Merge branch 'STREAMS-168'
Posted by sb...@apache.org.
Merge branch 'STREAMS-168'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ab9c6963
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ab9c6963
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ab9c6963
Branch: refs/heads/STREAMS-49
Commit: ab9c6963d8d3524ab88703a73a6c0e05842a5fee
Parents: e83659c d2ffe62
Author: sblackmon <sb...@apache.org>
Authored: Fri Nov 7 10:48:50 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Fri Nov 7 10:48:50 2014 -0800
----------------------------------------------------------------------
pom.xml | 2 +
streams-components/pom.xml | 62 +++++
streams-components/streams-http/README.md | 16 ++
streams-components/streams-http/pom.xml | 153 +++++++++++
.../components/http/HttpConfigurator.java | 62 +++++
.../http/processor/SimpleHTTPGetProcessor.java | 268 +++++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 230 ++++++++++++++++
.../components/http/HttpConfiguration.json | 50 ++++
.../http/HttpProcessorConfiguration.json | 28 ++
.../http/HttpProviderConfiguration.json | 18 ++
streams-contrib/pom.xml | 5 +-
.../streams-processor-peoplepattern/pom.xml | 138 ++++++++++
.../peoplepattern/AccountTypeProcessor.java | 75 ++++++
.../peoplepattern/DemographicsProcessor.java | 76 ++++++
.../streams/peoplepattern/AccountType.json | 27 ++
.../streams/peoplepattern/Demographics.json | 60 +++++
.../resources/templates/peoplepatternactor.json | 25 ++
.../api/FacebookPostActivitySerializer.java | 1 -
.../streams-provider-twitter/pom.xml | 5 +
.../processor/TwitterUrlApiProcessor.java | 73 +++++
.../provider/TwitterTimelineProvider.java | 7 +-
streams-pojo-extensions/pom.xml | 64 +++++
.../apache/streams/data/util/ExtensionUtil.java | 108 ++++++++
.../apache/streams/data/util/ActivityUtil.java | 14 +-
.../org/apache/streams/pojo/json/activity.json | 3 +-
.../org/apache/streams/pojo/json/object.json | 2 +-
26 files changed, 1559 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ab9c6963/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ab9c6963/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ab9c6963/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ab9c6963/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
[36/38] incubator-streams git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-streams
Posted by sb...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-streams
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/01716fe3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/01716fe3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/01716fe3
Branch: refs/heads/STREAMS-49
Commit: 01716fe31d6e7507b5a1aaec7e7060de0371c68b
Parents: 660f0c1 ab9c696
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Nov 7 13:13:50 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Nov 7 13:13:50 2014 -0600
----------------------------------------------------------------------
pom.xml | 58 ++++
streams-components/pom.xml | 62 +++++
streams-components/streams-http/README.md | 16 ++
streams-components/streams-http/pom.xml | 153 +++++++++++
.../components/http/HttpConfigurator.java | 62 +++++
.../http/processor/SimpleHTTPGetProcessor.java | 268 +++++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 230 ++++++++++++++++
.../components/http/HttpConfiguration.json | 50 ++++
.../http/HttpProcessorConfiguration.json | 28 ++
.../http/HttpProviderConfiguration.json | 18 ++
streams-contrib/pom.xml | 5 +-
streams-contrib/streams-persist-console/pom.xml | 34 +++
.../streams/console/ConsolePersistReader.java | 11 +-
.../streams/console/ConsolePersistWriter.java | 11 +-
.../streams-processor-peoplepattern/pom.xml | 138 ++++++++++
.../peoplepattern/AccountTypeProcessor.java | 75 ++++++
.../peoplepattern/DemographicsProcessor.java | 76 ++++++
.../streams/peoplepattern/AccountType.json | 27 ++
.../streams/peoplepattern/Demographics.json | 60 +++++
.../resources/templates/peoplepatternactor.json | 25 ++
.../api/FacebookPostActivitySerializer.java | 1 -
.../streams-provider-twitter/pom.xml | 5 +
.../processor/TwitterUrlApiProcessor.java | 73 +++++
.../provider/TwitterTimelineProvider.java | 7 +-
streams-pojo-extensions/pom.xml | 64 +++++
.../apache/streams/data/util/ExtensionUtil.java | 108 ++++++++
.../apache/streams/data/util/ActivityUtil.java | 14 +-
.../jackson/StreamsDateTimeDeserializer.java | 23 +-
.../streams/jackson/StreamsJacksonMapper.java | 20 ++
.../streams/jackson/StreamsJacksonModule.java | 9 +
.../org/apache/streams/pojo/json/activity.json | 3 +-
.../org/apache/streams/pojo/json/object.json | 2 +-
32 files changed, 1713 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
[33/38] incubator-streams git commit: adds arbitrary Joda format
support to StreamsJacksonMapper
Posted by sb...@apache.org.
adds arbitrary Joda format support to StreamsJacksonMapper
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e83659c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e83659c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e83659c5
Branch: refs/heads/STREAMS-49
Commit: e83659c51138b285dfa738739a765c7a222890c5
Parents: dbd7d89
Author: sblackmon <sb...@apache.org>
Authored: Thu Nov 6 13:58:11 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Fri Nov 7 10:47:21 2014 -0800
----------------------------------------------------------------------
.../jackson/StreamsDateTimeDeserializer.java | 23 +++++++++++++++++++-
.../streams/jackson/StreamsJacksonMapper.java | 20 +++++++++++++++++
.../streams/jackson/StreamsJacksonModule.java | 9 ++++++++
3 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e83659c5/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
index e0b98b2..8f53954 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
@@ -21,23 +21,44 @@ package org.apache.streams.jackson;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.google.common.collect.Lists;
import org.apache.streams.data.util.RFC3339Utils;
import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
/**
* Created by sblackmon on 3/27/14.
*/
public class StreamsDateTimeDeserializer extends StdDeserializer<DateTime> implements Serializable {
+ List<DateTimeFormatter> formatters = Lists.newArrayList();
+
protected StreamsDateTimeDeserializer(Class<DateTime> dateTimeClass) {
super(dateTimeClass);
}
+ protected StreamsDateTimeDeserializer(Class<DateTime> dateTimeClass, List<String> formats) {
+ super(dateTimeClass);
+ for( String format : formats )
+ formatters.add(DateTimeFormat.forPattern(format));
+ }
+
@Override
public DateTime deserialize(JsonParser jpar, DeserializationContext context) throws IOException {
- return RFC3339Utils.getInstance().parseToUTC(jpar.getValueAsString());
+
+ DateTime result = RFC3339Utils.parseToUTC(jpar.getValueAsString());
+ Iterator<DateTimeFormatter> iterator = formatters.iterator();
+ while( result == null && iterator.hasNext()) {
+ DateTimeFormatter formatter = iterator.next();
+ result = formatter.parseDateTime(jpar.getValueAsString());
+ }
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e83659c5/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
index 25c0c89..8a74caa 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
@@ -25,6 +25,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
+import java.util.List;
+
/**
* Created by sblackmon on 3/27/14.
*/
@@ -36,9 +38,27 @@ public class StreamsJacksonMapper extends ObjectMapper {
return INSTANCE;
}
+ public static StreamsJacksonMapper getInstance(List<String> formats){
+
+ StreamsJacksonMapper instance = new StreamsJacksonMapper(formats);
+
+ return instance;
+
+ }
+
public StreamsJacksonMapper() {
super();
registerModule(new StreamsJacksonModule());
+ configure();
+ }
+
+ public StreamsJacksonMapper(List<String> formats) {
+ super();
+ registerModule(new StreamsJacksonModule(formats));
+ configure();
+ }
+
+ public void configure() {
disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, Boolean.TRUE);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e83659c5/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
index 2869414..8b44b0f 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
@@ -22,6 +22,8 @@ import com.fasterxml.jackson.databind.module.SimpleModule;
import org.joda.time.DateTime;
import org.joda.time.Period;
+import java.util.List;
+
/**
* Created by sblackmon on 3/27/14.
*/
@@ -36,5 +38,12 @@ public class StreamsJacksonModule extends SimpleModule {
addDeserializer(Period.class, new StreamsPeriodDeserializer(Period.class));
}
+ public StreamsJacksonModule(List<String> formats) {
+ super();
+ addSerializer(DateTime.class, new StreamsDateTimeSerializer(DateTime.class));
+ addDeserializer(DateTime.class, new StreamsDateTimeDeserializer(DateTime.class, formats));
+ addSerializer(Period.class, new StreamsPeriodSerializer(Period.class));
+ addDeserializer(Period.class, new StreamsPeriodDeserializer(Period.class));
+ }
}
[26/38] incubator-streams git commit: simple Integration Test
Posted by sb...@apache.org.
simple Integration Test
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/adbb1b80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/adbb1b80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/adbb1b80
Branch: refs/heads/STREAMS-49
Commit: adbb1b80a52e1517d2833ad43ac1acad5d20610f
Parents: 9aebd0b
Author: sblackmon <sb...@apache.org>
Authored: Thu Nov 6 09:45:32 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Thu Nov 6 09:45:32 2014 -0800
----------------------------------------------------------------------
pom.xml | 56 ++++++++++++++++++++
streams-contrib/streams-persist-console/pom.xml | 34 ++++++++++++
.../streams/console/ConsolePersistReader.java | 11 ++--
.../streams/console/ConsolePersistWriter.java | 11 ++--
4 files changed, 103 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adbb1b80/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f8b6a14..e86f02d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,6 +74,8 @@
<jaxb2-basics.version>0.8.4</jaxb2-basics.version>
<jaxbutil.version>1.2.6</jaxbutil.version>
<junit.version>4.11</junit.version>
+ <surefire.plugin.version>2.17</surefire.plugin.version>
+ <failsafe.plugin.version>2.17</failsafe.plugin.version>
<slf4j.version>1.7.6</slf4j.version>
<logback.version>1.1.1</logback.version>
<commons-io.version>2.4</commons-io.version>
@@ -90,6 +92,7 @@
<facebook4j.version>2.1.0</facebook4j.version>
<maven.enforcer.plugin.version>1.3.1</maven.enforcer.plugin.version>
<mockito.version>1.9.5</mockito.version>
+ <powermock.version>1.5.6</powermock.version>
</properties>
<modules>
@@ -177,6 +180,41 @@
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper.version}</version>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${failsafe.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <!-- Sets the VM argument line used when integration tests are run. -->
+ <argLine>${failsafeArgLine}</argLine>
+ <!-- Skips integration tests if the value of skip.integration.tests property is true -->
+ <skipTests>${skip.integration.tests}</skipTests>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.plugin.version}</version>
+ <configuration>
+ <!-- Sets the VM argument line used when unit tests are run. -->
+ <argLine>${surefireArgLine}</argLine>
+ <!-- Skips unit tests if the value of skip.unit.tests property is true -->
+ <skipTests>${skip.unit.tests}</skipTests>
+ <!-- Excludes integration tests when unit tests are run. -->
+ <excludes>
+ <exclude>**/IT*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
</build>
@@ -231,6 +269,24 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adbb1b80/streams-contrib/streams-persist-console/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/pom.xml b/streams-contrib/streams-persist-console/pom.xml
index c7f2cd3..02ec403 100644
--- a/streams-contrib/streams-persist-console/pom.xml
+++ b/streams-contrib/streams-persist-console/pom.xml
@@ -26,5 +26,39 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-runtime-local</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ </dependency>
</dependencies>
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ </build>
+
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adbb1b80/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
index 776d5a3..8afba85 100644
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
+++ b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
@@ -31,6 +31,7 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.InputStream;
+import java.io.PrintStream;
import java.math.BigInteger;
import java.util.Queue;
import java.util.Scanner;
@@ -44,16 +45,16 @@ public class ConsolePersistReader implements StreamsPersistReader {
protected volatile Queue<StreamsDatum> persistQueue;
- private ObjectMapper mapper = new ObjectMapper();
+ protected InputStream inputStream = System.in;
public ConsolePersistReader() {
this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
}
- public ConsolePersistReader(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
+ public ConsolePersistReader(InputStream inputStream) {
+ this();
+ this.inputStream = inputStream;
}
-
public void prepare(Object o) {
}
@@ -77,7 +78,7 @@ public class ConsolePersistReader implements StreamsPersistReader {
LOGGER.info("{} readCurrent", STREAMS_ID);
- Scanner sc = new Scanner(System.in);
+ Scanner sc = new Scanner(inputStream);
while( sc.hasNextLine() ) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/adbb1b80/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
index 96d116f..53bb8d7 100644
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
+++ b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
@@ -27,6 +27,7 @@ import org.apache.streams.jackson.StreamsJacksonMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.PrintStream;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -34,6 +35,8 @@ public class ConsolePersistWriter implements StreamsPersistWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(ConsolePersistWriter.class);
+ protected PrintStream printStream = System.out;
+
protected volatile Queue<StreamsDatum> persistQueue;
private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
@@ -42,8 +45,9 @@ public class ConsolePersistWriter implements StreamsPersistWriter {
this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
}
- public ConsolePersistWriter(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
+ public ConsolePersistWriter(PrintStream printStream) {
+ this();
+ this.printStream = printStream;
}
public void prepare(Object o) {
@@ -61,8 +65,7 @@ public class ConsolePersistWriter implements StreamsPersistWriter {
String text = mapper.writeValueAsString(entry);
- System.out.println("\n"+text+"\n");
-// LOGGER.info(text);
+ printStream.println(text);
} catch (JsonProcessingException e) {
LOGGER.warn("save: {}", e);