You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/11/11 20:54:08 UTC
[01/32] incubator-streams git commit: just pons
Repository: incubator-streams
Updated Branches:
refs/heads/STREAMS-212 800bce964 -> 99e70b48c
just pons
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/1464819f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/1464819f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/1464819f
Branch: refs/heads/STREAMS-212
Commit: 1464819fd7ea147830a4211ea2ac6af4aa715022
Parents: 35a8fbf
Author: sblackmon <sb...@apache.org>
Authored: Sun Sep 14 10:24:47 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Sun Sep 14 10:24:47 2014 -0500
----------------------------------------------------------------------
streams-components/pom.xml | 62 ++++++++++
.../streams-processor-http/pom.xml | 114 +++++++++++++++++++
2 files changed, 176 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1464819f/streams-components/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/pom.xml b/streams-components/pom.xml
new file mode 100644
index 0000000..26384b1
--- /dev/null
+++ b/streams-components/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streams-project</artifactId>
+ <groupId>org.apache.streams</groupId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streams-components</artifactId>
+
+ <packaging>pom</packaging>
+ <name>streams-components</name>
+
+ <properties>
+
+ </properties>
+
+ <modules>
+ <module>streams-processor-http</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1464819f/streams-components/streams-processor-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/pom.xml b/streams-components/streams-processor-http/pom.xml
new file mode 100644
index 0000000..67538aa
--- /dev/null
+++ b/streams-components/streams-processor-http/pom.xml
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-project</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>streams-processor-http</artifactId>
+
+ <name>streams-processor-http</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-core</artifactId>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-source-jaxb2</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jaxb2</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>org.apache.streams.http</targetPackage>
+ <useLongIntegers>true</useLongIntegers>
+ <useJodaDates>true</useJodaDates>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
[26/32] incubator-streams git commit: Merge remote-tracking branch
'rbnks/STREAMS-199'
Posted by sb...@apache.org.
Merge remote-tracking branch 'rbnks/STREAMS-199'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/660f0c13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/660f0c13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/660f0c13
Branch: refs/heads/STREAMS-212
Commit: 660f0c13639003efd6a0355f25dc25ed66c4b8dc
Parents: b8e7a69 79eadea
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Nov 7 13:00:13 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Nov 7 13:00:13 2014 -0600
----------------------------------------------------------------------
.../local/builders/LocalStreamBuilder.java | 5 +++
.../streams/local/tasks/StreamsMergeTask.java | 7 +++++
.../local/tasks/StreamsPersistWriterTask.java | 18 ++++++++---
.../local/tasks/StreamsProcessorTask.java | 20 +++++++++---
.../local/tasks/StreamsProviderTask.java | 15 +++++++++
.../apache/streams/local/tasks/StreamsTask.java | 4 +++
.../local/builders/LocalStreamBuilderTest.java | 32 +++++++++++++++++---
.../streams/local/tasks/BasicTasksTest.java | 26 +++++++++++++---
8 files changed, 110 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[23/32] incubator-streams git commit: simple Integration Test
Posted by sb...@apache.org.
simple Integration Test
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/dbd7d890
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/dbd7d890
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/dbd7d890
Branch: refs/heads/STREAMS-212
Commit: dbd7d890804391025cf085f8a1c141a643d0b94b
Parents: b8e7a69
Author: sblackmon <sb...@apache.org>
Authored: Thu Nov 6 09:45:32 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Fri Nov 7 10:47:21 2014 -0800
----------------------------------------------------------------------
pom.xml | 56 ++++++++++++++++++++
streams-contrib/streams-persist-console/pom.xml | 34 ++++++++++++
.../streams/console/ConsolePersistReader.java | 11 ++--
.../streams/console/ConsolePersistWriter.java | 11 ++--
4 files changed, 103 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/dbd7d890/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f8b6a14..e86f02d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,6 +74,8 @@
<jaxb2-basics.version>0.8.4</jaxb2-basics.version>
<jaxbutil.version>1.2.6</jaxbutil.version>
<junit.version>4.11</junit.version>
+ <surefire.plugin.version>2.17</surefire.plugin.version>
+ <failsafe.plugin.version>2.17</failsafe.plugin.version>
<slf4j.version>1.7.6</slf4j.version>
<logback.version>1.1.1</logback.version>
<commons-io.version>2.4</commons-io.version>
@@ -90,6 +92,7 @@
<facebook4j.version>2.1.0</facebook4j.version>
<maven.enforcer.plugin.version>1.3.1</maven.enforcer.plugin.version>
<mockito.version>1.9.5</mockito.version>
+ <powermock.version>1.5.6</powermock.version>
</properties>
<modules>
@@ -177,6 +180,41 @@
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper.version}</version>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${failsafe.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <!-- Sets the VM argument line used when integration tests are run. -->
+ <argLine>${failsafeArgLine}</argLine>
+ <!-- Skips integration tests if the value of skip.integration.tests property is true -->
+ <skipTests>${skip.integration.tests}</skipTests>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.plugin.version}</version>
+ <configuration>
+ <!-- Sets the VM argument line used when unit tests are run. -->
+ <argLine>${surefireArgLine}</argLine>
+ <!-- Skips unit tests if the value of skip.unit.tests property is true -->
+ <skipTests>${skip.unit.tests}</skipTests>
+ <!-- Excludes integration tests when unit tests are run. -->
+ <excludes>
+ <exclude>**/IT*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
</build>
@@ -231,6 +269,24 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/dbd7d890/streams-contrib/streams-persist-console/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/pom.xml b/streams-contrib/streams-persist-console/pom.xml
index c7f2cd3..02ec403 100644
--- a/streams-contrib/streams-persist-console/pom.xml
+++ b/streams-contrib/streams-persist-console/pom.xml
@@ -26,5 +26,39 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-runtime-local</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ </dependency>
</dependencies>
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ </build>
+
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/dbd7d890/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
index 776d5a3..8afba85 100644
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
+++ b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
@@ -31,6 +31,7 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.InputStream;
+import java.io.PrintStream;
import java.math.BigInteger;
import java.util.Queue;
import java.util.Scanner;
@@ -44,16 +45,16 @@ public class ConsolePersistReader implements StreamsPersistReader {
protected volatile Queue<StreamsDatum> persistQueue;
- private ObjectMapper mapper = new ObjectMapper();
+ protected InputStream inputStream = System.in;
public ConsolePersistReader() {
this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
}
- public ConsolePersistReader(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
+ public ConsolePersistReader(InputStream inputStream) {
+ this();
+ this.inputStream = inputStream;
}
-
public void prepare(Object o) {
}
@@ -77,7 +78,7 @@ public class ConsolePersistReader implements StreamsPersistReader {
LOGGER.info("{} readCurrent", STREAMS_ID);
- Scanner sc = new Scanner(System.in);
+ Scanner sc = new Scanner(inputStream);
while( sc.hasNextLine() ) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/dbd7d890/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
index 96d116f..53bb8d7 100644
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
+++ b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
@@ -27,6 +27,7 @@ import org.apache.streams.jackson.StreamsJacksonMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.PrintStream;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -34,6 +35,8 @@ public class ConsolePersistWriter implements StreamsPersistWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(ConsolePersistWriter.class);
+ protected PrintStream printStream = System.out;
+
protected volatile Queue<StreamsDatum> persistQueue;
private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
@@ -42,8 +45,9 @@ public class ConsolePersistWriter implements StreamsPersistWriter {
this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
}
- public ConsolePersistWriter(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
+ public ConsolePersistWriter(PrintStream printStream) {
+ this();
+ this.printStream = printStream;
}
public void prepare(Object o) {
@@ -61,8 +65,7 @@ public class ConsolePersistWriter implements StreamsPersistWriter {
String text = mapper.writeValueAsString(entry);
- System.out.println("\n"+text+"\n");
-// LOGGER.info(text);
+ printStream.println(text);
} catch (JsonProcessingException e) {
LOGGER.warn("save: {}", e);
[12/32] incubator-streams git commit: Merge branch
'depricated_monitor' into countableImpl
Posted by sb...@apache.org.
Merge branch 'depricated_monitor' into countableImpl
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/015fadea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/015fadea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/015fadea
Branch: refs/heads/STREAMS-212
Commit: 015fadeac499777185f38056f2308272c8404794
Parents: d71568d 4f2e3cf
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Mon Oct 20 16:18:00 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Mon Oct 20 16:18:00 2014 -0500
----------------------------------------------------------------------
.../org/apache/streams/core/DatumStatusCountable.java | 1 +
.../java/org/apache/streams/core/DatumStatusCounter.java | 5 +++++
.../apache/streams/local/builders/StreamComponent.java | 1 +
.../streams/local/counters/StreamsTaskCounter.java | 11 ++++++++---
.../local/tasks/LocalStreamProcessMonitorThread.java | 1 +
.../local/tasks/StatusCounterMonitorRunnable.java | 1 +
.../streams/local/tasks/StatusCounterMonitorThread.java | 1 +
7 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[13/32] incubator-streams git commit: Merge pull request #11 from
apache/master
Posted by sb...@apache.org.
Merge pull request #11 from apache/master
Merge Apache master 2014/10/31
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/5c2e8322
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/5c2e8322
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/5c2e8322
Branch: refs/heads/STREAMS-212
Commit: 5c2e8322fb407df1d4059609a205569f71bc2b31
Parents: 2c71550 d88e8c8
Author: Ryan Ebanks <rb...@users.noreply.github.com>
Authored: Fri Oct 31 16:22:28 2014 -0500
Committer: Ryan Ebanks <rb...@users.noreply.github.com>
Committed: Fri Oct 31 16:22:28 2014 -0500
----------------------------------------------------------------------
streams-contrib/streams-amazon-aws/pom.xml | 2 +-
.../org/apache/streams/s3/S3Configurator.java | 55 +++----
.../org/apache/streams/s3/S3PersistReader.java | 11 +-
.../org/apache/streams/s3/S3PersistWriter.java | 10 +-
.../org/apache/streams/s3/S3Configuration.json | 18 ++-
.../google-gplus/pom.xml | 2 +-
.../processor/GooglePlusTypeConverter.java | 91 +++++++++++
.../util/GPlusPersonDeserializer.java | 107 +++++++++++++
.../serializer/util/GooglePlusActivityUtil.java | 115 ++++++++++++++
.../google/gplus/GooglePlusPersonSerDeTest.java | 97 ++++++++++++
.../processor/GooglePlusTypeConverterTest.java | 98 ++++++++++++
.../test/resources/google_plus_person_jsons.txt | 2 +
.../streams/core/DatumStatusCountable.java | 2 +
.../apache/streams/core/DatumStatusCounter.java | 7 +-
streams-runtimes/streams-runtime-local/pom.xml | 6 +-
.../streams/local/builders/StreamComponent.java | 1 +
.../local/counters/DatumStatusCounter.java | 83 ++++++++++
.../counters/DatumStatusCounterMXBean.java | 43 +++++
.../local/counters/StreamsTaskCounter.java | 157 ++++++++++++++++++
.../counters/StreamsTaskCounterMXBean.java | 63 ++++++++
...amOnUnhandleThrowableThreadPoolExecutor.java | 17 ++
.../tasks/LocalStreamProcessMonitorThread.java | 1 +
.../tasks/StatusCounterMonitorRunnable.java | 1 +
.../local/tasks/StatusCounterMonitorThread.java | 1 +
.../local/counters/DatumStatusCounterTest.java | 134 ++++++++++++++++
.../local/counters/StreamsTaskCounterTest.java | 158 +++++++++++++++++++
.../org/apache/streams/util/ComponentUtils.java | 19 ++-
27 files changed, 1253 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
[24/32] incubator-streams git commit: adds arbitrary Joda format
support to StreamsJacksonMapper
Posted by sb...@apache.org.
adds arbitrary Joda format support to StreamsJacksonMapper
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e83659c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e83659c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e83659c5
Branch: refs/heads/STREAMS-212
Commit: e83659c51138b285dfa738739a765c7a222890c5
Parents: dbd7d89
Author: sblackmon <sb...@apache.org>
Authored: Thu Nov 6 13:58:11 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Fri Nov 7 10:47:21 2014 -0800
----------------------------------------------------------------------
.../jackson/StreamsDateTimeDeserializer.java | 23 +++++++++++++++++++-
.../streams/jackson/StreamsJacksonMapper.java | 20 +++++++++++++++++
.../streams/jackson/StreamsJacksonModule.java | 9 ++++++++
3 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e83659c5/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
index e0b98b2..8f53954 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
@@ -21,23 +21,44 @@ package org.apache.streams.jackson;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.google.common.collect.Lists;
import org.apache.streams.data.util.RFC3339Utils;
import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
/**
* Created by sblackmon on 3/27/14.
*/
public class StreamsDateTimeDeserializer extends StdDeserializer<DateTime> implements Serializable {
+ List<DateTimeFormatter> formatters = Lists.newArrayList();
+
protected StreamsDateTimeDeserializer(Class<DateTime> dateTimeClass) {
super(dateTimeClass);
}
+ protected StreamsDateTimeDeserializer(Class<DateTime> dateTimeClass, List<String> formats) {
+ super(dateTimeClass);
+ for( String format : formats )
+ formatters.add(DateTimeFormat.forPattern(format));
+ }
+
@Override
public DateTime deserialize(JsonParser jpar, DeserializationContext context) throws IOException {
- return RFC3339Utils.getInstance().parseToUTC(jpar.getValueAsString());
+
+ DateTime result = RFC3339Utils.parseToUTC(jpar.getValueAsString());
+ Iterator<DateTimeFormatter> iterator = formatters.iterator();
+ while( result == null && iterator.hasNext()) {
+ DateTimeFormatter formatter = iterator.next();
+ result = formatter.parseDateTime(jpar.getValueAsString());
+ }
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e83659c5/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
index 25c0c89..8a74caa 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
@@ -25,6 +25,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
+import java.util.List;
+
/**
* Created by sblackmon on 3/27/14.
*/
@@ -36,9 +38,27 @@ public class StreamsJacksonMapper extends ObjectMapper {
return INSTANCE;
}
+ public static StreamsJacksonMapper getInstance(List<String> formats){
+
+ StreamsJacksonMapper instance = new StreamsJacksonMapper(formats);
+
+ return instance;
+
+ }
+
public StreamsJacksonMapper() {
super();
registerModule(new StreamsJacksonModule());
+ configure();
+ }
+
+ public StreamsJacksonMapper(List<String> formats) {
+ super();
+ registerModule(new StreamsJacksonModule(formats));
+ configure();
+ }
+
+ public void configure() {
disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, Boolean.TRUE);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e83659c5/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
index 2869414..8b44b0f 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
@@ -22,6 +22,8 @@ import com.fasterxml.jackson.databind.module.SimpleModule;
import org.joda.time.DateTime;
import org.joda.time.Period;
+import java.util.List;
+
/**
* Created by sblackmon on 3/27/14.
*/
@@ -36,5 +38,12 @@ public class StreamsJacksonModule extends SimpleModule {
addDeserializer(Period.class, new StreamsPeriodDeserializer(Period.class));
}
+ public StreamsJacksonModule(List<String> formats) {
+ super();
+ addSerializer(DateTime.class, new StreamsDateTimeSerializer(DateTime.class));
+ addDeserializer(DateTime.class, new StreamsDateTimeDeserializer(DateTime.class, formats));
+ addSerializer(Period.class, new StreamsPeriodSerializer(Period.class));
+ addDeserializer(Period.class, new StreamsPeriodDeserializer(Period.class));
+ }
}
[20/32] incubator-streams git commit: STREAMS-204 | Added test back
in with correct name
Posted by sb...@apache.org.
STREAMS-204 | Added test back in with correct name
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/b8e7a69d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/b8e7a69d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/b8e7a69d
Branch: refs/heads/STREAMS-212
Commit: b8e7a69d08edfa615c96722da30bf01f0b2d9b8b
Parents: e792bb9
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Thu Nov 6 14:11:00 2014 -0600
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Thu Nov 6 14:12:16 2014 -0600
----------------------------------------------------------------------
.../gplus/GooglePlusCommentSerDeTest.java | 114 +++++++++++++++++++
1 file changed, 114 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8e7a69d/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeTest.java
new file mode 100644
index 0000000..9fea22c
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.google.gplus;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Comment;
+import com.google.gplus.serializer.util.GPlusCommentDeserializer;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class GooglePlusCommentSerDeTest {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentSerDeTest.class);
+ private ObjectMapper objectMapper;
+ private GooglePlusActivityUtil googlePlusActivityUtil;
+
+ @Before
+ public void setup() {
+ objectMapper = new StreamsJacksonMapper();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Comment.class, new GPlusCommentDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ googlePlusActivityUtil = new GooglePlusActivityUtil();
+ }
+
+ @org.junit.Test
+ public void testCommentObjects() {
+ InputStream is = GooglePlusPersonSerDeTest.class.getResourceAsStream("/google_plus_comments_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ Activity activity = new Activity();
+ List<Comment> comments = Lists.newArrayList();
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Comment comment = objectMapper.readValue(line, Comment.class);
+
+ LOGGER.info("comment: {}", comment);
+
+ assertNotNull(comment);
+ assertNotNull(comment.getEtag());
+ assertNotNull(comment.getId());
+ assertNotNull(comment.getInReplyTo());
+ assertNotNull(comment.getObject());
+ assertNotNull(comment.getPlusoners());
+ assertNotNull(comment.getPublished());
+ assertNotNull(comment.getUpdated());
+ assertNotNull(comment.getSelfLink());
+ assertEquals(comment.getVerb(), "post");
+
+ comments.add(comment);
+ }
+ }
+
+ assertEquals(comments.size(), 3);
+
+ googlePlusActivityUtil.updateActivity(comments, activity);
+ assertNotNull(activity);
+ assertNotNull(activity.getObject());
+ assertEquals(activity.getObject().getAttachments().size(), 3);
+ } catch (Exception e) {
+ LOGGER.error("Exception while testing serializability: {}", e);
+ }
+ }
+
+ @org.junit.Test
+ public void testEmptyComments() {
+ Activity activity = new Activity();
+
+ googlePlusActivityUtil.updateActivity(new ArrayList<Comment>(), activity);
+
+ assertNull(activity.getObject());
+ }
+}
[21/32] incubator-streams git commit: Fixed Gplus provider and added
user data/activity collectors
Posted by sb...@apache.org.
Fixed Gplus provider and added user data/activity collectors
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/4e0e9dad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/4e0e9dad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/4e0e9dad
Branch: refs/heads/STREAMS-212
Commit: 4e0e9dad415d1767f0f6191377066be11bc645dc
Parents: 76763f8
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Thu Nov 6 17:08:34 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Thu Nov 6 17:08:34 2014 -0600
----------------------------------------------------------------------
.../google-gplus/pom.xml | 32 +++
.../gplus/provider/AbstractGPlusProvider.java | 234 ++++++++++++++++
.../gplus/provider/GPlusActivitySerializer.java | 4 +-
.../provider/GPlusHistoryProviderTask.java | 106 --------
.../google/gplus/provider/GPlusProvider.java | 189 -------------
.../provider/GPlusUserActivityCollector.java | 130 +++++++++
.../provider/GPlusUserActivityProvider.java | 18 ++
.../gplus/provider/GPlusUserDataCollector.java | 101 +++++++
.../gplus/provider/GPlusUserDataProvider.java | 18 ++
.../com/google/gplus/GPlusConfiguration.json | 44 ++-
.../provider/TestAbstractGPlusProvider.java | 79 ++++++
.../TestGPlusUserActivityCollector.java | 268 +++++++++++++++++++
.../provider/TestGPlusUserDataCollector.java | 131 +++++++++
streams-contrib/streams-provider-google/pom.xml | 6 +
.../backoff/AbstractBackOffStrategy.java | 15 +-
15 files changed, 1071 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/pom.xml b/streams-contrib/streams-provider-google/google-gplus/pom.xml
index 0437408..a734791 100644
--- a/streams-contrib/streams-provider-google/google-gplus/pom.xml
+++ b/streams-contrib/streams-provider-google/google-gplus/pom.xml
@@ -79,6 +79,38 @@
<artifactId>google-http-client-jackson2</artifactId>
<version>1.17.0-rc</version>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ <version>1.5.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>1.5.5</version>
+ <exclusions>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-reflect</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
new file mode 100644
index 0000000..daa34d5
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.google.gplus.provider;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.plus.Plus;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.typesafe.config.Config;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.google.gplus.GPlusConfiguration;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.ComponentUtils;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ *
+ */
+public abstract class AbstractGPlusProvider implements StreamsProvider {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(AbstractGPlusProvider.class);
+ private final static String SCOPE = "https://www.googleapis.com/auth/plus.stream.read";
+ private final static int MAX_BATCH_SIZE = 1000;
+
+ private static final HttpTransport TRANSPORT = new NetHttpTransport();
+ private static final JacksonFactory JSON_FACTORY = new JacksonFactory();
+ private static final Gson GSON = new Gson();
+
+ private GPlusConfiguration config;
+ private ExecutorService executor;
+ private BlockingQueue<StreamsDatum> datumQueue;
+ private BlockingQueue<Runnable> runnables;
+ private AtomicBoolean isComplete;
+ private boolean previousPullWasEmpty;
+
+ protected GoogleClientSecrets clientSecrets;
+ protected GoogleCredential credential;
+ protected Plus plus;
+
+
+
+ public AbstractGPlusProvider() {
+ Config config = StreamsConfigurator.config.getConfig("gplus");
+ this.config = GPlusConfigurator.detectConfiguration(config);
+ }
+
+ public AbstractGPlusProvider(GPlusConfiguration config) {
+ this.config = config;
+ }
+
+ @Override
+ public void startStream() {
+ BackOffStrategy backOffStrategy = new ExponentialBackOffStrategy(2);
+ for(UserInfo user : this.config.getGooglePlusUsers()) {
+ if(this.config.getDefaultAfterDate() != null && user.getAfterDate() == null) {
+ user.setAfterDate(this.config.getDefaultAfterDate());
+ }
+ if(this.config.getDefaultBeforeDate() != null && user.getBeforeDate() == null) {
+ user.setBeforeDate(this.config.getDefaultBeforeDate());
+ }
+ this.executor.submit(getDataCollector(backOffStrategy, this.datumQueue, this.plus, user));
+ }
+ this.executor.shutdown();
+ }
+
+ protected abstract Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo);
+
+ @Override
+ public StreamsResultSet readCurrent() {
+ BlockingQueue<StreamsDatum> batch = new LinkedBlockingQueue<>();
+ int batchCount = 0;
+ while(!this.datumQueue.isEmpty() && batchCount < MAX_BATCH_SIZE) {
+ StreamsDatum datum = ComponentUtils.pollWhileNotEmpty(this.datumQueue);
+ if(datum != null) {
+ ++batchCount;
+ ComponentUtils.offerUntilSuccess(datum, batch);
+ }
+ }
+ boolean pullIsEmpty = batch.isEmpty() && this.datumQueue.isEmpty() &&this.executor.isTerminated();
+ this.isComplete.set(this.previousPullWasEmpty && pullIsEmpty);
+ this.previousPullWasEmpty = pullIsEmpty;
+ return new StreamsResultSet(batch);
+ }
+
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ return null;
+ }
+
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ return null;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return !this.isComplete.get();
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ Preconditions.checkNotNull(config.getOauth().getConsumerKey());
+ Preconditions.checkNotNull(config.getOauth().getConsumerSecret());
+ Preconditions.checkNotNull(config.getOauth().getAccessToken());
+ Preconditions.checkNotNull(config.getOauth().getAccessTokenSecret());
+
+ try {
+ this.plus = createPlusClient();
+ } catch (IOException e) {
+ LOGGER.error("Failed to created oauth for GPlus : {}", e);
+ throw new RuntimeException(e);
+ }
+ // GPlus rate limits you to 5 calls per second, so there is not a need to execute more than one
+ // collector unless you have multiple oauth tokens
+ //TODO make this configurable based on the number of oauth tokens
+ this.executor = Executors.newFixedThreadPool(1);
+ this.datumQueue = new LinkedBlockingQueue<>(1000);
+ this.isComplete = new AtomicBoolean(false);
+ this.previousPullWasEmpty = false;
+ }
+
+ @VisibleForTesting
+ protected Plus createPlusClient() throws IOException{
+ credential = new GoogleCredential.Builder()
+ .setJsonFactory(JSON_FACTORY)
+ .setTransport(TRANSPORT)
+ .setClientSecrets(config.getOauth().getConsumerKey(), config.getOauth().getConsumerSecret()).build()
+ .setFromTokenResponse(JSON_FACTORY.fromString(
+ config.getOauth().getAccessToken(), GoogleTokenResponse.class));
+ credential.refreshToken();
+ return new Plus.Builder(TRANSPORT,JSON_FACTORY, credential).build();
+ }
+
+ @Override
+ public void cleanUp() {
+ ComponentUtils.shutdownExecutor(this.executor, 10, 10);
+ this.executor = null;
+ }
+
+ public GPlusConfiguration getConfig() {
+ return config;
+ }
+
+ public void setConfig(GPlusConfiguration config) {
+ this.config = config;
+ }
+
+ /**
+ * Set and overwrite the default before date that was read from the configuration file.
+ * @param defaultBeforeDate
+ */
+ public void setDefaultBeforeDate(DateTime defaultBeforeDate) {
+ this.config.setDefaultBeforeDate(defaultBeforeDate);
+ }
+
+ /**
+ * Set and overwrite the default after date that was read from teh configuration file.
+ * @param defaultAfterDate
+ */
+ public void setDefaultAfterDate(DateTime defaultAfterDate) {
+ this.config.setDefaultAfterDate(defaultAfterDate);
+ }
+
+ /**
+ * Sets and overwrite the user info from the configuaration file. Uses the defaults before and after dates.
+ * @param userIds
+ */
+ public void setUserInfoWithDefaultDates(Set<String> userIds) {
+ List<UserInfo> gPlusUsers = Lists.newLinkedList();
+ for(String userId : userIds) {
+ UserInfo user = new UserInfo();
+ user.setUserId(userId);
+ user.setAfterDate(this.config.getDefaultAfterDate());
+ user.setBeforeDate(this.config.getDefaultBeforeDate());
+ gPlusUsers.add(user);
+ }
+ this.config.setGooglePlusUsers(gPlusUsers);
+ }
+
+ /**
+ * Set and overwrite user into from teh configuration file. Only sets after dater.
+ * @param usersAndAfterDates
+ */
+ public void setUserInfoWithAfterDate(Map<String, DateTime> usersAndAfterDates) {
+ List<UserInfo> gPlusUsers = Lists.newLinkedList();
+ for(String userId : usersAndAfterDates.keySet()) {
+ UserInfo user = new UserInfo();
+ user.setUserId(userId);
+ user.setAfterDate(usersAndAfterDates.get(userId));
+ gPlusUsers.add(user);
+ }
+ this.config.setGooglePlusUsers(gPlusUsers);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
index 1659ae3..4991e94 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
@@ -32,9 +32,9 @@ public class GPlusActivitySerializer implements ActivitySerializer<com.google.ap
private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivitySerializer.class);
- GPlusProvider provider;
+ AbstractGPlusProvider provider;
- public GPlusActivitySerializer(GPlusProvider provider) {
+ public GPlusActivitySerializer(AbstractGPlusProvider provider) {
this.provider = provider;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusHistoryProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusHistoryProviderTask.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusHistoryProviderTask.java
deleted file mode 100644
index 46eb301..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusHistoryProviderTask.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.services.plus.Plus;
-import com.google.api.services.plus.model.ActivityFeed;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Created by sblackmon on 12/10/13.
- */
-public class GPlusHistoryProviderTask implements Runnable {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GPlusHistoryProviderTask.class);
-
- private ObjectMapper mapper;
-
- private GPlusProvider provider;
- private String userid;
- private String circle;
-
- public GPlusHistoryProviderTask(GPlusProvider provider, String userid, String circle) {
- this.provider = provider;
- this.userid = userid;
- this.circle = circle;
- }
-
- @Override
- public void run() {
-
- Plus.Activities.List listActivities = null;
- try {
- listActivities = provider.plus.activities().list(userid, circle);
- } catch (IOException e) {
- e.printStackTrace();
- return;
- }
- listActivities.setMaxResults(100L);
-
-// Execute the request for the first page
- ActivityFeed activityFeed = null;
- try {
- activityFeed = listActivities.execute();
- } catch (IOException e) {
- e.printStackTrace();
- return;
- }
-
-// Unwrap the request and extract the pieces we want
- List<com.google.api.services.plus.model.Activity> activities = activityFeed.getItems();
-
-// Loop through until we arrive at an empty page
- while (activities != null) {
- for (com.google.api.services.plus.model.Activity gplusActivity : activities) {
- String json = null;
- try {
- json = mapper.writeValueAsString(gplusActivity);
- } catch (JsonProcessingException e) {
- e.printStackTrace();
- }
- provider.inQueue.offer(json);
- }
-
- // We will know we are on the last page when the next page token is null.
- // If this is the case, break.
- if (activityFeed.getNextPageToken() == null) {
- break;
- }
-
- // Prepare to request the next page of activities
- listActivities.setPageToken(activityFeed.getNextPageToken());
-
- // Execute and process the next page request
- try {
- activityFeed = listActivities.execute();
- } catch (IOException e) {
- e.printStackTrace();
- }
- activities = activityFeed.getItems();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusProvider.java
deleted file mode 100644
index 9257783..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusProvider.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.javanet.NetHttpTransport;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.plus.Plus;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.gson.Gson;
-import com.typesafe.config.Config;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.google.gplus.GPlusConfiguration;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.util.Queue;
-import java.util.Random;
-import java.util.concurrent.*;
-
-/**
- * Created by sblackmon on 12/10/13.
- */
-public class GPlusProvider implements StreamsProvider {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GPlusProvider.class);
-
- private GPlusConfiguration config;
-
- private Class klass;
-
- public GPlusConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(GPlusConfiguration config) {
- this.config = config;
- }
-
- protected BlockingQueue inQueue = new LinkedBlockingQueue<String>(10000);
-
- protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<StreamsDatum>();
-
- public BlockingQueue<Object> getInQueue() {
- return inQueue;
- }
-
- private static final HttpTransport TRANSPORT = new NetHttpTransport();
- private static final JacksonFactory JSON_FACTORY = new JacksonFactory();
- private static final Gson GSON = new Gson();
-
- protected GoogleClientSecrets clientSecrets;
- protected GoogleCredential credential;
- protected Plus plus;
-
- protected ListeningExecutorService executor = MoreExecutors.listeningDecorator(newFixedThreadPoolWithQueueSize(5, 20));
-
- ListenableFuture providerTaskComplete;
-
- private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
- return new ThreadPoolExecutor(nThreads, nThreads,
- 5000L, TimeUnit.MILLISECONDS,
- new ArrayBlockingQueue<Runnable>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
- }
-
- public GPlusProvider() {
- Config config = StreamsConfigurator.config.getConfig("gplus");
- this.config = GPlusConfigurator.detectConfiguration(config);
- }
-
- public GPlusProvider(GPlusConfiguration config) {
- this.config = config;
- }
-
- public GPlusProvider(Class klass) {
- Config config = StreamsConfigurator.config.getConfig("gplus");
- this.config = GPlusConfigurator.detectConfiguration(config);
- this.klass = klass;
- }
-
- public GPlusProvider(GPlusConfiguration config, Class klass) {
- this.config = config;
- this.klass = klass;
- }
-
- @Override
- public void startStream() {
-
- providerTaskComplete = executor.submit(new GPlusHistoryProviderTask(this, "me", "public"));
-
- for (int i = 0; i < 1; i++) {
- new Thread(new GPlusEventProcessor(inQueue, providerQueue, klass));
- }
-
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- startStream();
-
- while( !providerTaskComplete.isDone()) {
- try {
- Thread.sleep(new Random().nextInt(100));
- } catch (InterruptedException e) { }
- }
-
- return new StreamsResultSet(providerQueue);
-
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !providerTaskComplete.isDone() && !providerTaskComplete.isCancelled();
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- Preconditions.checkNotNull(this.klass);
-
- Preconditions.checkNotNull(config.getOauth().getConsumerKey());
- Preconditions.checkNotNull(config.getOauth().getConsumerSecret());
- Preconditions.checkNotNull(config.getOauth().getAccessToken());
- Preconditions.checkNotNull(config.getOauth().getAccessTokenSecret());
-
- try {
- credential = new GoogleCredential.Builder()
- .setJsonFactory(JSON_FACTORY)
- .setTransport(TRANSPORT)
- .setClientSecrets(config.getOauth().getConsumerKey(), config.getOauth().getConsumerSecret()).build()
- .setFromTokenResponse(JSON_FACTORY.fromString(
- config.getOauth().getAccessToken(), GoogleTokenResponse.class));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public void cleanUp() {
- for (int i = 0; i < 1; i++) {
- inQueue.add(GPlusEventProcessor.TERMINATE);
- }
-
- try {
- executor.awaitTermination(5, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
new file mode 100644
index 0000000..c6952bf
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
@@ -0,0 +1,130 @@
+package com.google.gplus.provider;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.services.plus.Plus;
+import com.google.api.services.plus.model.Activity;
+import com.google.api.services.plus.model.ActivityFeed;
+import com.google.gplus.serializer.util.GPlusActivityDeserializer;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffException;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingQueue;
+
+/**
+ *
+ */
+public class GPlusUserActivityCollector implements Runnable {
+
+ /**
+ * Key for all public activities
+ * https://developers.google.com/+/api/latest/activities/list
+ */
+ private static final String PUBLIC_COLLECTION = "public";
+ /**
+ * Max results allowed per request
+ * https://developers.google.com/+/api/latest/activities/list
+ */
+ private static final long MAX_RESULTS = 100;
+ private static final int MAX_ATTEMPTS = 5;
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserActivityCollector.class);
+ private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+
+ static { //set up mapper for Google Activity Object
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Activity.class, new GPlusActivityDeserializer());
+ MAPPER.registerModule(simpleModule);
+ MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ private BlockingQueue<StreamsDatum> datumQueue;
+ private BackOffStrategy backOff;
+ private Plus gPlus;
+ private UserInfo userInfo;
+
+ public GPlusUserActivityCollector(Plus gPlus, BlockingQueue<StreamsDatum> datumQueue, BackOffStrategy backOff, UserInfo userInfo) {
+ this.gPlus = gPlus;
+ this.datumQueue = datumQueue;
+ this.backOff = backOff;
+ this.userInfo = userInfo;
+ }
+
+ @Override
+ public void run() {
+ collectActivityData();
+ }
+
+ protected void collectActivityData() {
+ try {
+ ActivityFeed feed = null;
+ boolean tryAgain = false;
+ int attempt = 0;
+ DateTime afterDate = userInfo.getAfterDate();
+ DateTime beforeDate = userInfo.getBeforeDate();
+ do {
+ try {
+ if(feed == null) {
+ feed = this.gPlus.activities().list(this.userInfo.getUserId(), PUBLIC_COLLECTION).setMaxResults(MAX_RESULTS).execute();
+ } else {
+ feed = this.gPlus.activities().list(this.userInfo.getUserId(), PUBLIC_COLLECTION).setMaxResults(MAX_RESULTS).setPageToken(feed.getNextPageToken()).execute();
+ }
+ this.backOff.reset(); //successful pull reset api.
+ for(com.google.api.services.plus.model.Activity activity : feed.getItems()) {
+ DateTime published = new DateTime(activity.getPublished().getValue());
+ if( (afterDate == null && beforeDate == null)
+ || (beforeDate == null && afterDate.isBefore(published))
+ || (afterDate == null && beforeDate.isAfter(published))
+ || ((afterDate != null && beforeDate != null) && (afterDate.isBefore(published) && beforeDate.isAfter(published)))) {
+ this.datumQueue.put(new StreamsDatum(MAPPER.writeValueAsString(activity), activity.getId()));
+ } else if(afterDate != null && afterDate.isAfter(published)) {
+ feed.setNextPageToken(null); // do not fetch next page
+ break;
+ }
+ }
+ } catch (GoogleJsonResponseException gjre) {
+ switch (gjre.getStatusCode()) {
+ case 400 :
+ LOGGER.warn("Bad Request for user={} : {}", userInfo.getUserId(), gjre);
+ tryAgain = false;
+ break;
+ case 401 :
+ LOGGER.warn("Invalid Credentials : {}", gjre);
+ tryAgain = false;
+ case 403 :
+ LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
+ this.backOff.backOff();
+ tryAgain = true;
+ break;
+ case 503 :
+ LOGGER.warn("Google Backend Service Error : {}", gjre);
+ tryAgain = false;
+ break;
+ default:
+ LOGGER.warn("Google Service returned error : {}", gjre);
+ tryAgain = true;
+ this.backOff.backOff();
+ break;
+ }
+ ++attempt;
+ }
+ } while((tryAgain || (feed != null && feed.getNextPageToken() != null)) && attempt < MAX_ATTEMPTS);
+ } catch (Throwable t) {
+ if(t instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ t.printStackTrace();
+ LOGGER.warn("Unable to pull Activities for user={} : {}",this.userInfo.getUserId(), t);
+ }
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityProvider.java
new file mode 100644
index 0000000..e7f1bec
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityProvider.java
@@ -0,0 +1,18 @@
+package com.google.gplus.provider;
+
+import com.google.api.services.plus.Plus;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+
+import java.util.concurrent.BlockingQueue;
+
+/**
+ *
+ */
+public class GPlusUserActivityProvider extends AbstractGPlusProvider{
+ @Override
+ protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
+ return new GPlusUserActivityCollector(plus, queue, strategy, userInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
new file mode 100644
index 0000000..6a5ce49
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
@@ -0,0 +1,101 @@
+package com.google.gplus.provider;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.services.plus.Plus;
+import com.google.api.services.plus.model.Person;
+import com.google.gplus.serializer.util.GPlusPersonDeserializer;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingQueue;
+
+/**
+ *
+ */
+public class GPlusUserDataCollector implements Runnable{
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserDataCollector.class);
+ private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+ private static final int MAX_ATTEMPTS = 5;
+
+ static { //set up Mapper for Person objects
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
+ MAPPER.registerModule(simpleModule);
+ MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ private BackOffStrategy backOffStrategy;
+ private Plus gPlus;
+ private BlockingQueue<StreamsDatum> datumQueue;
+ private UserInfo userInfo;
+
+
+ public GPlusUserDataCollector(Plus gPlus, BackOffStrategy backOffStrategy, BlockingQueue<StreamsDatum> datumQueue, UserInfo userInfo) {
+ this.gPlus = gPlus;
+ this.backOffStrategy = backOffStrategy;
+ this.datumQueue = datumQueue;
+ this.userInfo = userInfo;
+ }
+
+ protected void queueUserHistory() {
+ try {
+ boolean tryAgain = false;
+ int attempts = 0;
+ com.google.api.services.plus.model.Person person = null;
+ do {
+ try {
+ person = this.gPlus.people().get(userInfo.getUserId()).execute();
+ this.backOffStrategy.reset();
+ tryAgain = person == null;
+ } catch (GoogleJsonResponseException gjre) {
+ switch (gjre.getStatusCode()) {
+ case 400 :
+ LOGGER.warn("Bad Request for user={} : {}", userInfo.getUserId(), gjre);
+ tryAgain = false;
+ break;
+ case 401 :
+ LOGGER.warn("Invalid Credentials : {}", gjre);
+ tryAgain = false;
+ case 403 :
+ LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
+ this.backOffStrategy.backOff();
+ tryAgain = true;
+ break;
+ case 503 :
+ LOGGER.warn("Google Backend Service Error : {}", gjre);
+ tryAgain = false;
+ break;
+ default:
+ LOGGER.warn("Google Service returned error : {}", gjre);
+ tryAgain = true;
+ this.backOffStrategy.backOff();
+ break;
+ }
+ }
+ ++attempts;
+ } while(tryAgain && attempts < MAX_ATTEMPTS);
+ this.datumQueue.put(new StreamsDatum(MAPPER.writeValueAsString(person), person.getId()));
+ } catch (Throwable t) {
+ LOGGER.warn("Unable to pull user data for user={} : {}", userInfo.getUserId(), t);
+ if(t instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ queueUserHistory();
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataProvider.java
new file mode 100644
index 0000000..0e2782d
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataProvider.java
@@ -0,0 +1,18 @@
+package com.google.gplus.provider;
+
+import com.google.api.services.plus.Plus;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+
+import java.util.concurrent.BlockingQueue;
+
+/**
+ *
+ */
+public class GPlusUserDataProvider extends AbstractGPlusProvider{
+ @Override
+ protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
+ return new GPlusUserDataCollector(plus, strategy, queue, userInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
index e2d8130..32f6a72 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
@@ -26,11 +26,29 @@
},
"follow": {
"type": "array",
- "description": "A list of user names, indicating the users whose activities should be delivered on the stream",
+ "description": "DEPRECATED. A list of user names, indicating the users whose activities should be delivered on the stream",
"items": {
"type": "string"
}
},
+ "googlePlusUsers": {
+ "type": "array",
+ "description": "A list of user user ids and optional date parameters for the GPlus provider",
+ "items": {
+ "type": "object",
+ "$ref": "#/definitions/userInfo"
+ }
+ },
+ "defaultAfterDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "Optional parameter for the provider. If this value is not null an the afterDate value in the userInfo is null, this value will be used."
+ },
+ "defaultBeforeDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "Optional parameter for the provider. If this value is not null and the beforeDate value in the userInfo is null, this value will be used."
+ },
"oauth": {
"type": "object",
"dynamic": "true",
@@ -54,5 +72,29 @@
}
}
}
+ },
+ "definitions": {
+ "userInfo": {
+ "type": "object",
+ "javaInterfaces" : ["java.io.Serializable"],
+ "dynamic": "true",
+ "javaType": "org.apache.streams.google.gplus.configuration.UserInfo",
+ "properties": {
+ "userId": {
+ "type": "string",
+ "description": "instagram user id"
+ },
+ "afterDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for this user. If this is null it will use the defaultBeforeDate."
+ },
+ "beforeDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for this user.. If this is null it will use the defaultAfterDate."
+ }
+ }
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
new file mode 100644
index 0000000..80b1b23
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
@@ -0,0 +1,79 @@
+package com.google.gplus.provider;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.google.api.services.plus.Plus;
+import com.google.common.collect.Lists;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.GPlusConfiguration;
+import org.apache.streams.google.gplus.GPlusOAuthConfiguration;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+/**
+ *
+ */
+public class TestAbstractGPlusProvider extends RandomizedTest{
+
+ @Test
+ @Repeat(iterations = 3)
+ public void testDataCollectorRunsPerUser() {
+ int numUsers = randomIntBetween(1, 1000);
+ List<UserInfo> userList = Lists.newLinkedList();
+ for(int i=0; i < numUsers; ++i) {
+ userList.add(new UserInfo());
+ }
+ GPlusConfiguration config = new GPlusConfiguration();
+ GPlusOAuthConfiguration oauth = new GPlusOAuthConfiguration();
+ oauth.setAccessToken("a");
+ oauth.setConsumerKey("a");
+ oauth.setConsumerSecret("a");
+ oauth.setAccessTokenSecret("a");
+ config.setOauth(oauth);
+ config.setGooglePlusUsers(userList);
+ AbstractGPlusProvider provider = new AbstractGPlusProvider(config) {
+
+ @Override
+ protected Plus createPlusClient() throws IOException {
+ return mock(Plus.class);
+ }
+
+ @Override
+ protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
+ final BlockingQueue<StreamsDatum> q = queue;
+ return new Runnable() {
+ @Override
+ public void run() {
+ try {
+ q.put(new StreamsDatum(null));
+ } catch (InterruptedException ie) {
+ fail("Test was interrupted");
+ }
+ }
+ };
+ }
+ };
+
+ try {
+ provider.prepare(null);
+ provider.startStream();
+ int datumCount = 0;
+ while(provider.isRunning()) {
+ datumCount += provider.readCurrent().size();
+ }
+ assertEquals(numUsers, datumCount);
+ } finally {
+ provider.cleanUp();
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserActivityCollector.java
new file mode 100644
index 0000000..1484af5
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserActivityCollector.java
@@ -0,0 +1,268 @@
+package com.google.gplus.provider;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.plus.Plus;
+import com.google.api.services.plus.model.Activity;
+import com.google.api.services.plus.model.ActivityFeed;
+import com.google.common.collect.Lists;
+import com.google.gplus.serializer.util.GPlusActivityDeserializer;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.util.api.requests.backoff.impl.ConstantTimeBackOffStrategy;
+import org.joda.time.DateTime;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for {@link com.google.gplus.provider.GPlusUserActivityCollector}
+ */
+public class TestGPlusUserActivityCollector extends RandomizedTest {
+
+
+ private static final String ACTIVITY_TEMPLATE = "{ \"kind\": \"plus#activity\", \"etag\": \"\\\"Vea_b94Y77GDGgRK7gFNPnolKQw/v1-6aVSBGT4qiStMoz7f2_AN2fM\\\"\", \"title\": \"\", \"published\": \"%s\", \"updated\": \"2014-10-27T06:26:33.927Z\", \"id\": \"z13twrlznpvtzz52w22mdt1y0k3of1djw04\", \"url\": \"https://plus.google.com/116771159471120611293/posts/GR7CGR8N5VL\", \"actor\": { \"id\": \"116771159471120611293\", \"displayName\": \"Matt Neithercott\", \"url\": \"https://plus.google.com/116771159471120611293\", \"image\": { \"url\": \"https://lh6.googleusercontent.com/-C0fiZBxdvw0/AAAAAAAAAAI/AAAAAAAAJ5k/K4pgR3_-_ms/photo.jpg?sz=50\" } }, \"verb\": \"share\", \"object\": { \"objectType\": \"activity\", \"id\": \"z13zgvtiurjgfti1v234iflghvq2c1dge04\", \"actor\": { \"id\": \"104954254300557350002\", \"displayName\": \"Adam Balm\", \"url\": \"https://plus.google.com/104954254300557350002\", \"image\": { \"url\": \"https://lh4.googleusercontent.com/-SO1scj4p2LA/AAAAAAAAAAI/AAAAAAAAI-
s/efA9LBVe144/photo.jpg?sz=50\" } }, \"content\": \"\", \"url\": \"https://plus.google.com/104954254300557350002/posts/AwewXhtn7ws\", \"replies\": { \"totalItems\": 0, \"selfLink\": \"https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/comments\" }, \"plusoners\": { \"totalItems\": 9, \"selfLink\": \"https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/people/plusoners\" }, \"resharers\": { \"totalItems\": 0, \"selfLink\": \"https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/people/resharers\" }, \"attachments\": [ { \"objectType\": \"photo\", \"id\": \"104954254300557350002.6074732746360957410\", \"content\": \"26/10/2014 - 1\", \"url\": \"https://plus.google.com/photos/104954254300557350002/albums/6074732747132702225/6074732746360957410\", \"image\": { \"url\": \"https://lh4.googleusercontent.com/-oO3fnARlDm0/VE3JP1xHKeI/AAAAAAAAeCY/-X2jzc6HruA/w506-h750/2014%2B-%2B1\", \"type\":
\"image/jpeg\" }, \"fullImage\": { \"url\": \"https://lh4.googleusercontent.com/-oO3fnARlDm0/VE3JP1xHKeI/AAAAAAAAeCY/-X2jzc6HruA/w600-h1141/2014%2B-%2B1\", \"type\": \"image/jpeg\", \"height\": 1141, \"width\": 600 } } ] }, \"annotation\": \"Truth 😜\", \"provider\": { \"title\": \"Reshared Post\" }, \"access\": { \"kind\": \"plus#acl\", \"description\": \"Public\", \"items\": [ { \"type\": \"public\" } ] } }";
+ private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+ private static final String IN_RANGE_IDENTIFIER = "data in range";
+
+
+ static {
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Activity.class, new GPlusActivityDeserializer());
+ MAPPER.registerModule(simpleModule);
+ MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ /**
+ * Creates a randomized activity and randomized date range.
+ * The activity feed is separated into three chunks,
+ * |. . . data too recent to be in date range . . .||. . . data in date range. . .||. . . data too old to be in date range|
+ * [index 0, ............................................................................................., index length-1]
+ * Inside of those chunks data has no order, but the list is ordered by those three chunks.
+ *
+ * The test will check to see if the num of data in the date range make onto the output queue.
+ */
+ @Test
+ @Repeat(iterations = 3)
+ public void testWithBeforeAndAfterDates() throws InterruptedException {
+ //initialize counts assuming no date ranges will be used
+ int numActivities = randomIntBetween(0, 1000);
+ int numActivitiesInDateRange = numActivities;
+ int numberOutOfRange = 0;
+ int numBerforeRange = 0;
+ int numAfterRange = 0;
+ //determine if date ranges will be used
+ DateTime beforeDate = null;
+ DateTime afterDate = null;
+ if(randomInt() % 2 == 0) {
+ beforeDate = DateTime.now().minusDays(randomIntBetween(1,5));
+ }
+ if(randomInt() % 2 == 0) {
+ if(beforeDate == null) {
+ afterDate = DateTime.now().minusDays(randomIntBetween(1, 10));
+ } else {
+ afterDate = beforeDate.minusDays(randomIntBetween(1, 10));
+ }
+ }
+ //update counts if date ranges are going to be used.
+ if(beforeDate != null || afterDate != null ) { //assign amount to be in range
+ numActivitiesInDateRange = randomIntBetween(0, numActivities);
+ numberOutOfRange = numActivities - numActivitiesInDateRange;
+ }
+ if(beforeDate == null && afterDate != null) { //assign all out of range to be before the start of the range
+ numBerforeRange = numberOutOfRange;
+ } else if(beforeDate != null && afterDate == null) { //assign all out of range to be after the start of the range
+ numAfterRange = numberOutOfRange;
+ } else if(beforeDate != null && afterDate != null) { //assign half before range and half after the range
+ numAfterRange = (numberOutOfRange / 2) + (numberOutOfRange % 2);
+ numBerforeRange = numberOutOfRange / 2;
+ }
+
+ Plus plus = createMockPlus(numBerforeRange, numAfterRange, numActivitiesInDateRange, afterDate, beforeDate);
+ BackOffStrategy strategy = new ConstantTimeBackOffStrategy(1);
+ BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
+ UserInfo userInfo = new UserInfo();
+ userInfo.setUserId("A");
+ userInfo.setAfterDate(afterDate);
+ userInfo.setBeforeDate(beforeDate);
+ GPlusUserActivityCollector collector = new GPlusUserActivityCollector(plus, datums, strategy, userInfo);
+ collector.run();
+
+ assertEquals(numActivitiesInDateRange, datums.size());
+ while(!datums.isEmpty()) {
+ StreamsDatum datum = datums.take();
+ assertNotNull(datum);
+ assertNotNull(datum.getDocument());
+ assertTrue(datum.getDocument() instanceof String);
+ assertTrue(((String)datum.getDocument()).contains(IN_RANGE_IDENTIFIER)); //only in range documents are on the out going queue.
+ }
+ }
+
+
+ private Plus createMockPlus(final int numBefore, final int numAfter, final int numInRange, final DateTime after, final DateTime before) {
+ Plus plus = mock(Plus.class);
+ final Plus.Activities activities = createMockPlusActivities(numBefore, numAfter, numInRange, after, before);
+ doAnswer(new Answer() {
+ @Override
+ public Plus.Activities answer(InvocationOnMock invocationOnMock) throws Throwable {
+ return activities;
+ }
+ }).when(plus).activities();
+ return plus;
+ }
+
+ private Plus.Activities createMockPlusActivities(final int numBefore, final int numAfter, final int numInRange, final DateTime after, final DateTime before) {
+ Plus.Activities activities = mock(Plus.Activities.class);
+ try {
+ Plus.Activities.List list = createMockPlusActivitiesList(numBefore, numAfter, numInRange, after, before);
+ when(activities.list(anyString(), anyString())).thenReturn(list);
+ } catch (IOException ioe) {
+ fail("Should not have thrown exception while creating mock. : "+ioe.getMessage());
+ }
+ return activities;
+ }
+
+ private Plus.Activities.List createMockPlusActivitiesList(final int numBefore, final int numAfter, final int numInRange, final DateTime after, final DateTime before) {
+ Plus.Activities.List list = mock(Plus.Activities.List.class);
+ when(list.setMaxResults(anyLong())).thenReturn(list);
+ when(list.setPageToken(anyString())).thenReturn(list);
+ ActivityFeedAnswer answer = new ActivityFeedAnswer(numBefore, numAfter, numInRange, after, before);
+ try {
+ doAnswer(answer).when(list).execute();
+ } catch (IOException ioe) {
+ fail("Should not have thrown exception while creating mock. : "+ioe.getMessage());
+ }
+ return list;
+ }
+
+
+ private static ActivityFeed createMockActivityFeed(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before, boolean page) {
+ ActivityFeed feed = new ActivityFeed();
+ List<Activity> list = Lists.newLinkedList();
+ for(int i=0; i < numAfter; ++i) {
+ DateTime published = before.plus(randomIntBetween(0, Integer.MAX_VALUE));
+ Activity activity = createActivityWithPublishedDate(published);
+ list.add(activity);
+ }
+ for(int i=0; i < numInRange; ++i) {
+ DateTime published = null;
+ if((before == null && after == null) || before == null) {
+ published = DateTime.now(); // no date range or end time date range so just make the time now.
+ } else if(after == null) {
+ published = before.minusMillis(randomIntBetween(1, Integer.MAX_VALUE)); //no beginning to range
+ } else { // has to be in range
+ long range = before.getMillis() - after.getMillis();
+ published = after.plus(range / 2); //in the middle
+ }
+ Activity activity = createActivityWithPublishedDate(published);
+ activity.setTitle(IN_RANGE_IDENTIFIER);
+ list.add(activity);
+ }
+ for(int i=0; i < numBefore; ++i) {
+ DateTime published = after.minusMillis(randomIntBetween(1, Integer.MAX_VALUE));
+ Activity activity = createActivityWithPublishedDate(published);
+ list.add(activity);
+ }
+ if(page) {
+ feed.setNextPageToken("A");
+ } else {
+ feed.setNextPageToken(null);
+ }
+ feed.setItems(list);
+ return feed;
+ }
+
+ private static Activity createActivityWithPublishedDate(DateTime dateTime) {
+ Activity activity = new Activity();
+ activity.setPublished(new com.google.api.client.util.DateTime(dateTime.getMillis()));
+ activity.setId("a");
+ return activity;
+ }
+
+ private static class ActivityFeedAnswer implements Answer<ActivityFeed> {
+ private int afterCount = 0;
+ private int beforeCount = 0;
+ private int inCount = 0;
+ private int maxBatch = 100;
+
+ private int numAfter;
+ private int numInRange;
+ private int numBefore;
+ private DateTime after;
+ private DateTime before;
+
+ private ActivityFeedAnswer(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
+ this.numBefore = numBefore;
+ this.numAfter = numAfter;
+ this.numInRange = numInRange;
+ this.after = after;
+ this.before = before;
+ }
+
+
+
+
+ @Override
+ public ActivityFeed answer(InvocationOnMock invocationOnMock) throws Throwable {
+ int totalCount = 0;
+ int batchAfter = 0;
+ int batchBefore = 0;
+ int batchIn = 0;
+ if(afterCount != numAfter) {
+ if(numAfter - afterCount >= maxBatch) {
+ afterCount += maxBatch;
+ batchAfter += maxBatch;
+ totalCount += batchAfter;
+ } else {
+ batchAfter += numAfter - afterCount;
+ totalCount += numAfter - afterCount;
+ afterCount = numAfter;
+ }
+ }
+ if(totalCount < maxBatch && inCount != numInRange) {
+ if(numInRange - inCount >= maxBatch - totalCount) {
+ inCount += maxBatch - totalCount;
+ batchIn += maxBatch - totalCount;
+ totalCount += batchIn;
+ } else {
+ batchIn += numInRange - inCount;
+ totalCount += numInRange - inCount;
+ inCount = numInRange;
+ }
+ }
+ if(totalCount < maxBatch && beforeCount != numBefore) {
+ if(numBefore - batchBefore >= maxBatch - totalCount) {
+ batchBefore += maxBatch - totalCount;
+ totalCount = maxBatch;
+ beforeCount +=batchBefore;
+ } else {
+ batchBefore += numBefore - beforeCount;
+ totalCount += numBefore - beforeCount;
+ beforeCount = numBefore;
+ }
+ }
+
+ return createMockActivityFeed(batchBefore, batchAfter, batchIn, after, before, numAfter != afterCount || inCount != numInRange || beforeCount != numBefore);
+ }
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserDataCollector.java
new file mode 100644
index 0000000..784e1b5
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserDataCollector.java
@@ -0,0 +1,131 @@
+package com.google.gplus.provider;
+
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.services.plus.Plus;
+import com.google.api.services.plus.model.Person;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.util.api.requests.backoff.impl.ConstantTimeBackOffStrategy;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Basic Units for {@link com.google.gplus.provider.GPlusUserDataCollector}
+ */
+public class TestGPlusUserDataCollector {
+
+ private static final String NO_ERROR = "no error";
+
+
+ /**
+ * Test that on success a datum will be added to the queue.
+ * @throws Exception
+ */
+ @Test
+ public void testSucessfullPull() throws Exception {
+ Plus plus = createMockPlus(0, null);
+ BackOffStrategy backOff = new ConstantTimeBackOffStrategy(1);
+ BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
+ UserInfo user = new UserInfo();
+ user.setUserId("A");
+
+ GPlusUserDataCollector collector = new GPlusUserDataCollector(plus, backOff, datums, user);
+ collector.run();
+
+ assertEquals(1, datums.size());
+ StreamsDatum datum = datums.take();
+ assertNotNull(datum);
+ assertEquals(NO_ERROR, datum.getId());
+ assertNotNull(datum.getDocument());
+ assertTrue(datum.getDocument() instanceof String);
+ }
+
+ /**
+ * Test that on failure, no datums are output
+ * @throws Exception
+ */
+ @Test
+ public void testFail() throws Exception {
+ Plus plus = createMockPlus(3, mock(GoogleJsonResponseException.class));
+ UserInfo user = new UserInfo();
+ user.setUserId("A");
+ BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
+ BackOffStrategy backOffStrategy = new ConstantTimeBackOffStrategy(1);
+
+ GPlusUserDataCollector collector = new GPlusUserDataCollector(plus, backOffStrategy, datums, user);
+ collector.run();
+
+ assertEquals(0, datums.size());
+ }
+
+
+
+ private Plus createMockPlus(final int succedOnTry, final Throwable throwable) {
+ Plus plus = mock(Plus.class);
+ doAnswer(new Answer() {
+ @Override
+ public Plus.People answer(InvocationOnMock invocationOnMock) throws Throwable {
+ return createMockPeople(succedOnTry, throwable);
+ }
+ }).when(plus).people();
+ return plus;
+ }
+
+ private Plus.People createMockPeople(final int succedOnTry, final Throwable throwable) {
+ Plus.People people = mock(Plus.People.class);
+ try {
+ when(people.get(anyString())).thenAnswer(new Answer<Plus.People.Get>() {
+ @Override
+ public Plus.People.Get answer(InvocationOnMock invocationOnMock) throws Throwable {
+ return createMockGetNoError(succedOnTry, throwable);
+ }
+ });
+ } catch (IOException ioe) {
+ fail("No Excpetion should have been thrown while creating mocks");
+ }
+ return people;
+ }
+
+ private Plus.People.Get createMockGetNoError(final int succedOnTry, final Throwable throwable) {
+ Plus.People.Get get = mock(Plus.People.Get.class);
+ try {
+ doAnswer(new Answer() {
+ private int counter =0;
+
+ @Override
+ public Person answer(InvocationOnMock invocationOnMock) throws Throwable {
+ if(counter == succedOnTry) {
+ Person p = new Person();
+ p.setId(NO_ERROR);
+ return p;
+ } else {
+ ++counter;
+ throw throwable;
+ }
+ }
+ }).when(get).execute();
+ } catch (IOException ioe) {
+ fail("No Excpetion should have been thrown while creating mocks");
+ }
+ return get;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-contrib/streams-provider-google/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/pom.xml b/streams-contrib/streams-provider-google/pom.xml
index b720b76..99cd158 100644
--- a/streams-contrib/streams-provider-google/pom.xml
+++ b/streams-contrib/streams-provider-google/pom.xml
@@ -39,6 +39,12 @@
<artifactId>streams-pojo</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4e0e9dad/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/AbstractBackOffStrategy.java
----------------------------------------------------------------------
diff --git a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/AbstractBackOffStrategy.java b/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/AbstractBackOffStrategy.java
index 45e4239..a68a94a 100644
--- a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/AbstractBackOffStrategy.java
+++ b/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/AbstractBackOffStrategy.java
@@ -14,6 +14,8 @@ specific language governing permissions and limitations
under the License. */
package org.apache.streams.util.api.requests.backoff;
+import java.util.concurrent.atomic.AtomicInteger;
+
/**
* @see org.apache.streams.util.api.requests.backoff.BackOffStrategy
*/
@@ -22,7 +24,7 @@ public abstract class AbstractBackOffStrategy implements BackOffStrategy {
private long baseSleepTime;
private long lastSleepTime;
private int maxAttempts;
- private int attemptsCount;
+ private AtomicInteger attemptsCount;
/**
* A BackOffStrategy that can effectively be used endlessly.
@@ -46,16 +48,17 @@ public abstract class AbstractBackOffStrategy implements BackOffStrategy {
}
this.baseSleepTime = baseBackOffTime;
this.maxAttempts = maximumNumberOfBackOffAttempts;
- this.attemptsCount = 0;
+ this.attemptsCount = new AtomicInteger(0);
}
@Override
public void backOff() throws BackOffException {
- if(this.attemptsCount++ >= this.maxAttempts && this.maxAttempts != -1) {
- throw new BackOffException(this.attemptsCount-1, this.lastSleepTime);
+ int attempt = this.attemptsCount.getAndIncrement();
+ if(attempt >= this.maxAttempts && this.maxAttempts != -1) {
+ throw new BackOffException(attempt, this.lastSleepTime);
} else {
try {
- Thread.sleep(this.lastSleepTime = calculateBackOffTime(this.attemptsCount, this.baseSleepTime));
+ Thread.sleep(this.lastSleepTime = calculateBackOffTime(attempt, this.baseSleepTime));
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
@@ -64,7 +67,7 @@ public abstract class AbstractBackOffStrategy implements BackOffStrategy {
@Override
public void reset() {
- this.attemptsCount = 0;
+ this.attemptsCount.set(0);
}
/**
[28/32] incubator-streams git commit: Added abstract class to reduce
repeative code
Posted by sb...@apache.org.
Added abstract class to reduce repeative code
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/4d6d02a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/4d6d02a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/4d6d02a0
Branch: refs/heads/STREAMS-212
Commit: 4d6d02a09d0cb708bb98fe17af44c82b71b0530c
Parents: ff6cb52
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Nov 7 14:39:12 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Nov 7 14:39:12 2014 -0600
----------------------------------------------------------------------
.../gplus/provider/GPlusDataCollector.java | 50 ++++++++++++++++++++
.../provider/GPlusUserActivityCollector.java | 26 +---------
.../gplus/provider/GPlusUserDataCollector.java | 26 +---------
.../com/google/gplus/GPlusConfiguration.json | 2 +-
4 files changed, 55 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4d6d02a0/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java
new file mode 100644
index 0000000..1894dc4
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java
@@ -0,0 +1,50 @@
+package com.google.gplus.provider;
+
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import org.apache.streams.util.api.requests.backoff.BackOffException;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public abstract class GPlusDataCollector implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusDataCollector.class);
+
+
+ /**
+ * Looks at the status code of the expception. If the code indicates that the request should be retried,
+ * it executes the back off strategy and returns true.
+ * @param gjre
+ * @param backOff
+ * @return returns true if the error code of the exception indicates the request should be retried.
+ */
+ public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException {
+ boolean tryAgain = false;
+ switch (gjre.getStatusCode()) {
+ case 400 :
+ LOGGER.warn("Bad Request : {}", gjre);
+ break;
+ case 401 :
+ LOGGER.warn("Invalid Credentials : {}", gjre);
+ case 403 :
+ LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
+ backOff.backOff();
+ tryAgain = true;
+ break;
+ case 503 :
+ LOGGER.warn("Google Backend Service Error : {}", gjre);
+ break;
+ default:
+ LOGGER.warn("Google Service returned error : {}", gjre);
+ tryAgain = true;
+ backOff.backOff();
+ break;
+ }
+ return tryAgain;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4d6d02a0/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
index d9a89d9..04f0aef 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
@@ -22,7 +22,7 @@ import java.util.concurrent.BlockingQueue;
/**
* Collects the public activities of a GPlus user. Has ability to filter by date ranges.
*/
-public class GPlusUserActivityCollector implements Runnable {
+public class GPlusUserActivityCollector extends GPlusDataCollector {
/**
* Key for all public activities
@@ -90,29 +90,7 @@ public class GPlusUserActivityCollector implements Runnable {
}
}
} catch (GoogleJsonResponseException gjre) {
- switch (gjre.getStatusCode()) {
- case 400 :
- LOGGER.warn("Bad Request for user={} : {}", userInfo.getUserId(), gjre);
- tryAgain = false;
- break;
- case 401 :
- LOGGER.warn("Invalid Credentials : {}", gjre);
- tryAgain = false;
- case 403 :
- LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
- this.backOff.backOff();
- tryAgain = true;
- break;
- case 503 :
- LOGGER.warn("Google Backend Service Error : {}", gjre);
- tryAgain = false;
- break;
- default:
- LOGGER.warn("Google Service returned error : {}", gjre);
- tryAgain = true;
- this.backOff.backOff();
- break;
- }
+ tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOff);
++attempt;
}
} while((tryAgain || (feed != null && feed.getNextPageToken() != null)) && attempt < MAX_ATTEMPTS);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4d6d02a0/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
index 5269e10..65ef9b1 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
@@ -19,7 +19,7 @@ import java.util.concurrent.BlockingQueue;
/**
* Collects user profile information for a specific GPlus user
*/
-public class GPlusUserDataCollector implements Runnable{
+public class GPlusUserDataCollector extends GPlusDataCollector {
private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserDataCollector.class);
private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
@@ -56,29 +56,7 @@ public class GPlusUserDataCollector implements Runnable{
this.backOffStrategy.reset();
tryAgain = person == null;
} catch (GoogleJsonResponseException gjre) {
- switch (gjre.getStatusCode()) {
- case 400 :
- LOGGER.warn("Bad Request for user={} : {}", userInfo.getUserId(), gjre);
- tryAgain = false;
- break;
- case 401 :
- LOGGER.warn("Invalid Credentials : {}", gjre);
- tryAgain = false;
- case 403 :
- LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
- this.backOffStrategy.backOff();
- tryAgain = true;
- break;
- case 503 :
- LOGGER.warn("Google Backend Service Error : {}", gjre);
- tryAgain = false;
- break;
- default:
- LOGGER.warn("Google Service returned error : {}", gjre);
- tryAgain = true;
- this.backOffStrategy.backOff();
- break;
- }
+ tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOffStrategy);
}
++attempts;
} while(tryAgain && attempts < MAX_ATTEMPTS);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4d6d02a0/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
index 32f6a72..87ada8b 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
@@ -82,7 +82,7 @@
"properties": {
"userId": {
"type": "string",
- "description": "instagram user id"
+ "description": "Google+ user id"
},
"afterDate": {
"type": "string",
[10/32] incubator-streams git commit: Implemented new
StreamTasksCounter into StreamsTasks
Posted by sb...@apache.org.
Implemented new StreamTasksCounter into StreamsTasks
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/7e65a423
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/7e65a423
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/7e65a423
Branch: refs/heads/STREAMS-212
Commit: 7e65a423f31c2fc4540b72ade6be904c77638784
Parents: d305371
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Mon Oct 20 14:16:40 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Mon Oct 20 14:16:40 2014 -0500
----------------------------------------------------------------------
.../streams/local/tasks/StreamsMergeTask.java | 7 ++++
.../local/tasks/StreamsPersistWriterTask.java | 18 +++++++--
.../local/tasks/StreamsProcessorTask.java | 20 ++++++++--
.../local/tasks/StreamsProviderTask.java | 15 +++++++
.../apache/streams/local/tasks/StreamsTask.java | 4 ++
.../streams/local/tasks/BasicTasksTest.java | 41 ++++++++++++++++++--
6 files changed, 93 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
index 7a4c806..8280f29 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
@@ -19,6 +19,8 @@
package org.apache.streams.local.tasks;
import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.local.counters.StreamsTaskCounter;
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -78,4 +80,9 @@ public class StreamsMergeTask extends BaseStreamsTask {
}
}
}
+
+ @Override
+ public void setStreamsTaskCounter(StreamsTaskCounter counter) {
+ throw new NotImplementedException();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
index cab46b8..003ab9e 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
@@ -20,13 +20,11 @@ package org.apache.streams.local.tasks;
import org.apache.streams.core.*;
import org.apache.streams.core.util.DatumUtils;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
+import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -45,6 +43,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
private BlockingQueue<StreamsDatum> inQueue;
private AtomicBoolean isRunning;
private AtomicBoolean blocked;
+ private StreamsTaskCounter counter;
private DatumStatusCounter statusCounter = new DatumStatusCounter();
@@ -99,6 +98,9 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
public void run() {
try {
this.writer.prepare(this.streamConfig);
+ if(this.counter == null) {
+ this.counter = new StreamsTaskCounter(this.writer.getClass().getName()+ UUID.randomUUID().toString());
+ }
while(this.keepRunning.get()) {
StreamsDatum datum = null;
try {
@@ -111,14 +113,18 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
Thread.currentThread().interrupt();
}
if(datum != null) {
+ this.counter.incrementReceivedCount();
try {
+ long startTime = System.currentTimeMillis();
this.writer.write(datum);
+ this.counter.addTime(System.currentTimeMillis() - startTime);
statusCounter.incrementStatus(DatumStatus.SUCCESS);
} catch (Exception e) {
LOGGER.error("Error writing to persist writer {}", this.writer.getClass().getSimpleName(), e);
this.keepRunning.set(false); // why do we shutdown on a failed write ?
statusCounter.incrementStatus(DatumStatus.FAIL);
DatumUtils.addErrorToMetadata(datum, e, this.writer.getClass());
+ this.counter.incrementErrorCount();
}
} else { //datums should never be null
LOGGER.debug("Received null StreamsDatum @ writer : {}", this.writer.getClass().getName());
@@ -151,4 +157,8 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
return queues;
}
+ @Override
+ public void setStreamsTaskCounter(StreamsTaskCounter counter) {
+ this.counter = counter;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index ee69127..8d66847 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -21,13 +21,11 @@ package org.apache.streams.local.tasks;
import com.google.common.collect.Maps;
import org.apache.streams.core.*;
import org.apache.streams.core.util.DatumUtils;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
+import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -47,6 +45,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
private BlockingQueue<StreamsDatum> inQueue;
private AtomicBoolean isRunning;
private AtomicBoolean blocked;
+ private StreamsTaskCounter counter;
private DatumStatusCounter statusCounter = new DatumStatusCounter();
@@ -105,6 +104,9 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
public void run() {
try {
this.processor.prepare(this.streamConfig);
+ if(this.counter == null) {
+ this.counter = new StreamsTaskCounter(this.processor.getClass().getName()+ UUID.randomUUID().toString());
+ }
while(this.keepRunning.get()) {
StreamsDatum datum = null;
try {
@@ -117,11 +119,15 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
Thread.currentThread().interrupt();
}
if(datum != null) {
+ this.counter.incrementReceivedCount();
try {
+ long startTime = System.currentTimeMillis();
List<StreamsDatum> output = this.processor.process(datum);
+ this.counter.addTime(System.currentTimeMillis() - startTime);
if(output != null) {
for(StreamsDatum outDatum : output) {
super.addToOutgoingQueue(datum);
+ this.counter.incrementEmittedCount();
statusCounter.incrementStatus(DatumStatus.SUCCESS);
}
}
@@ -130,6 +136,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
this.keepRunning.set(false);
Thread.currentThread().interrupt();
} catch (Throwable t) {
+ this.counter.incrementErrorCount();
LOGGER.warn("Caught Throwable in processor, {} : {}", this.processor.getClass().getName(), t.getMessage());
statusCounter.incrementStatus(DatumStatus.FAIL);
//Add the error to the metadata, but keep processing
@@ -151,4 +158,9 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
queues.add(this.inQueue);
return queues;
}
+
+ @Override
+ public void setStreamsTaskCounter(StreamsTaskCounter counter) {
+ this.counter = counter;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index c16f64d..2475780 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -20,6 +20,7 @@ package org.apache.streams.local.tasks;
import org.apache.streams.core.*;
import org.apache.streams.core.util.DatumUtils;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,6 +28,7 @@ import org.slf4j.LoggerFactory;
import java.math.BigInteger;
import java.util.Map;
import java.util.Queue;
+import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -64,6 +66,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
private long sleepTime;
private int zeros = 0;
private DatumStatusCounter statusCounter = new DatumStatusCounter();
+ private StreamsTaskCounter counter;
/**
* Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readCurrent()}
@@ -145,13 +148,18 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
StreamsResultSet resultSet = null;
//Negative values mean we want to run forever
long maxZeros = timeout < 0 ? Long.MAX_VALUE : (timeout / sleepTime);
+ if(this.counter == null) { //should never be null
+ this.counter = new StreamsTaskCounter(this.provider.getClass().getName()+ UUID.randomUUID().toString());
+ }
switch(this.type) {
case PERPETUAL: {
provider.startStream();
this.started.set(true);
while(this.isRunning()) {
try {
+ long startTime = System.currentTimeMillis();
resultSet = provider.readCurrent();
+ this.counter.addTime(System.currentTimeMillis() - startTime);
if( resultSet.size() == 0 )
zeros++;
else {
@@ -164,6 +172,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
if(zeros > 0)
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
+ this.counter.incrementErrorCount();
LOGGER.warn("Thread interrupted");
this.keepRunning.set(false);
}
@@ -219,8 +228,10 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
if(datum != null) {
try {
super.addToOutgoingQueue(datum);
+ this.counter.incrementEmittedCount();
statusCounter.incrementStatus(DatumStatus.SUCCESS);
} catch( Exception e ) {
+ this.counter.incrementErrorCount();
statusCounter.incrementStatus(DatumStatus.FAIL);
DatumUtils.addErrorToMetadata(datum, e, this.provider.getClass());
}
@@ -229,4 +240,8 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
this.flushing.set(false);
}
+ @Override
+ public void setStreamsTaskCounter(StreamsTaskCounter counter) {
+ this.counter = counter;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
index 7513631..8423095 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
@@ -19,6 +19,7 @@
package org.apache.streams.local.tasks;
import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import java.util.List;
import java.util.Map;
@@ -87,4 +88,7 @@ public interface StreamsTask extends Runnable{
*/
public List<BlockingQueue<StreamsDatum>> getOutputQueues();
+
+ public void setStreamsTaskCounter(StreamsTaskCounter counter);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e65a423/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
index f524db0..f62250d 100644
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
@@ -19,12 +19,18 @@
package org.apache.streams.local.tasks;
import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.local.counters.DatumStatusCounter;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.streams.local.queues.ThroughputQueue;
import org.apache.streams.local.test.processors.PassthroughDatumCounterProcessor;
import org.apache.streams.local.test.providers.NumericMessageProvider;
import org.apache.streams.local.test.writer.DatumCounterWriter;
+import org.junit.After;
import org.junit.Test;
+import javax.management.InstanceNotFoundException;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
import java.util.Queue;
import java.util.concurrent.*;
@@ -36,6 +42,21 @@ import static org.junit.Assert.*;
public class BasicTasksTest {
+ private static final String MBEAN_ID = "test_bean";
+
+ /**
+ * Remove registered mbeans from previous tests
+ * @throws Exception
+ */
+ @After
+ public void unregisterMXBean() throws Exception {
+ try {
+ ManagementFactory.getPlatformMBeanServer().unregisterMBean(new ObjectName(String.format(StreamsTaskCounter.NAME_TEMPLATE, MBEAN_ID)));
+ } catch (InstanceNotFoundException ife) {
+ //No-op
+ }
+ }
+
@Test
public void testProviderTask() {
@@ -77,7 +98,7 @@ public class BasicTasksTest {
assertTrue("Task should have completed running in aloted time.", service.isTerminated());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
- };
+ }
}
@Test
@@ -85,6 +106,8 @@ public class BasicTasksTest {
int numMessages = 100;
PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor("");
StreamsProcessorTask task = new StreamsProcessorTask(processor);
+ StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID);
+ task.setStreamsTaskCounter(counter);
BlockingQueue<StreamsDatum> outQueue = new LinkedBlockingQueue<>();
BlockingQueue<StreamsDatum> inQueue = createInputQueue(numMessages);
task.addOutputQueue(outQueue);
@@ -104,8 +127,7 @@ public class BasicTasksTest {
fail("Processor task failed to output "+numMessages+" in a timely fashion.");
}
}
- task.stopTask();
- assertEquals(numMessages, processor.getMessageCount());
+ task.stopTask();;
service.shutdown();
try {
if(!service.awaitTermination(5, TimeUnit.SECONDS)){
@@ -116,6 +138,11 @@ public class BasicTasksTest {
} catch (InterruptedException e) {
fail("Test Interupted.");
}
+ assertEquals(numMessages, processor.getMessageCount());
+ assertEquals(numMessages, counter.getNumReceived());
+ assertEquals(numMessages, counter.getNumEmitted());
+ assertEquals(0, counter.getNumUnhandledErrors());
+ assertEquals(0.0, counter.getErrorRate(), 0.0);
}
@Test
@@ -123,6 +150,8 @@ public class BasicTasksTest {
int numMessages = 100;
DatumCounterWriter writer = new DatumCounterWriter("");
StreamsPersistWriterTask task = new StreamsPersistWriterTask(writer);
+ StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID);
+ task.setStreamsTaskCounter(counter);
BlockingQueue<StreamsDatum> outQueue = new LinkedBlockingQueue<>();
BlockingQueue<StreamsDatum> inQueue = createInputQueue(numMessages);
@@ -150,7 +179,6 @@ public class BasicTasksTest {
}
}
task.stopTask();
- assertEquals(numMessages, writer.getDatumsCounted());
service.shutdown();
try {
if(!service.awaitTermination(5, TimeUnit.SECONDS)){
@@ -161,6 +189,11 @@ public class BasicTasksTest {
} catch (InterruptedException e) {
fail("Test Interupted.");
}
+ assertEquals(numMessages, writer.getDatumsCounted());
+ assertEquals(numMessages, counter.getNumReceived());
+ assertEquals(0, counter.getNumEmitted());
+ assertEquals(0, counter.getNumUnhandledErrors());
+ assertEquals(0.0, counter.getErrorRate(), 0.0);
}
@Test
[09/32] incubator-streams git commit: added license header and
javadoc comments
Posted by sb...@apache.org.
added license header and javadoc comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d2ffe62b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d2ffe62b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d2ffe62b
Branch: refs/heads/STREAMS-212
Commit: d2ffe62baf88618c7c8a6b9a1e5a79d21ed9e3fd
Parents: 0c8e67c
Author: sblackmon <sb...@apache.org>
Authored: Mon Oct 13 17:07:19 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Mon Oct 13 17:07:19 2014 -0500
----------------------------------------------------------------------
.../components/http/HttpConfigurator.java | 2 +-
.../http/processor/SimpleHTTPGetProcessor.java | 18 +++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 18 +++++++++++++++++
.../peoplepattern/AccountTypeProcessor.java | 2 +-
.../processor/TwitterUrlApiProcessor.java | 20 ++++++++++++++++++-
.../apache/streams/data/util/ExtensionUtil.java | 21 ++++++++++++++++++++
6 files changed, 78 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
index 900831f..979a680 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
@@ -25,7 +25,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
+ * Converts a {@link com.typesafe.config.Config} element into an instance of HttpConfiguration
*/
public class HttpConfigurator {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
index d3d4429..b8c957c 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.components.http.processor;
import com.fasterxml.jackson.core.JsonProcessingException;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
index 36084c7..118d06b 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.components.http.provider;
import com.fasterxml.jackson.databind.JsonNode;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
index edcf4d3..4e4a6af 100644
--- a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
import java.util.Map;
/**
- * Enrich actor with demographics
+ * Enrich actor with account type
*/
public class AccountTypeProcessor extends SimpleHTTPGetProcessor {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
index 54e1369..17ce411 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.twitter.processor;
import com.google.common.base.Preconditions;
@@ -13,7 +31,7 @@ import java.util.List;
import java.util.Map;
/**
- * Created by sblackmon on 9/14/14.
+ * Class gets a global share count from Twitter API for links on Activity datums
*/
public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements StreamsProcessor {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d2ffe62b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
index 7ce013c..1e0e384 100644
--- a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
+++ b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.data.util;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -7,6 +25,9 @@ import org.apache.streams.pojo.json.ActivityObject;
import java.util.Map;
+/**
+ * Class makes it easier to manage extensions added to activities, actors, objects, etc...
+ */
public class ExtensionUtil {
/**
[30/32] incubator-streams git commit: add license header
Posted by sb...@apache.org.
add license header
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/a6f86266
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/a6f86266
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/a6f86266
Branch: refs/heads/STREAMS-212
Commit: a6f8626621d2ecb34071628d957e6c69673c2597
Parents: 800bce9
Author: sblackmon <sb...@apache.org>
Authored: Tue Nov 11 12:55:02 2014 -0600
Committer: sblackmon <sb...@apache.org>
Committed: Tue Nov 11 12:55:02 2014 -0600
----------------------------------------------------------------------
.../serializer/DatasiftEventClassifier.java | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/a6f86266/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
index e90de6a..0169f17 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.streams.datasift.serializer;
import org.apache.streams.data.ActivitySerializer;
[17/32] incubator-streams git commit: STREAMS-204 | Initial commit
Posted by sb...@apache.org.
STREAMS-204 | Initial commit
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/2358b4e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/2358b4e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/2358b4e7
Branch: refs/heads/STREAMS-212
Commit: 2358b4e76d3b0f47d07eb569ef6affefbea0dee0
Parents: 9aebd0b
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Wed Oct 29 17:42:11 2014 -0500
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Thu Nov 6 14:04:13 2014 -0600
----------------------------------------------------------------------
.../google/gplus/processor/GooglePlusCommentProcessor.java | 7 +++++++
.../gplus/serializer/util/GPlusCommentDeserializer.java | 7 +++++++
.../java/com/google/gplus/GooglePLusCommentSerDeTest.java | 7 +++++++
.../src/test/resources/google_plus_comments_jsons.txt | 0
4 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2358b4e7/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
new file mode 100644
index 0000000..7dc4d56
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
@@ -0,0 +1,7 @@
+package com.google.gplus.processor;
+
+/**
+ * Created by rdouglas on 10/29/14.
+ */
+public class GooglePlusCommentProcessor {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2358b4e7/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
new file mode 100644
index 0000000..51446e0
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
@@ -0,0 +1,7 @@
+package com.google.gplus.serializer.util;
+
+/**
+ * Created by rdouglas on 10/29/14.
+ */
+public class GPlusCommentDeserializer {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2358b4e7/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
new file mode 100644
index 0000000..b3479f9
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
@@ -0,0 +1,7 @@
+package com.google.gplus;
+
+/**
+ * Created by rdouglas on 10/29/14.
+ */
+public class GooglePLusCommentSerDeTest {
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/2358b4e7/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
new file mode 100644
index 0000000..e69de29
[03/32] incubator-streams git commit: added HttpConfigurator
Posted by sb...@apache.org.
added HttpConfigurator
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/b8ccf9f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/b8ccf9f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/b8ccf9f6
Branch: refs/heads/STREAMS-212
Commit: b8ccf9f6869c64f1f6e25f668942ea0de9fec2eb
Parents: 34232ad
Author: sblackmon <sb...@apache.org>
Authored: Mon Sep 15 18:38:09 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Mon Sep 15 18:38:09 2014 -0500
----------------------------------------------------------------------
.../components/http/HttpConfigurator.java | 53 ++++++++++++++++++++
.../components/http/SimpleHTTPGetProcessor.java | 23 +++++----
.../processor/TwitterUrlApiProcessor.java | 8 +++
3 files changed, 75 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8ccf9f6/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
new file mode 100644
index 0000000..36801b8
--- /dev/null
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.components.http;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigRenderOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
+ */
+public class HttpConfigurator {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(HttpConfigurator.class);
+
+ private final static ObjectMapper mapper = new ObjectMapper();
+
+ public static HttpProcessorConfiguration detectConfiguration(Config config) {
+
+ HttpProcessorConfiguration httpProcessorConfiguration = null;
+
+ try {
+ httpProcessorConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), HttpProcessorConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse http configuration", e.getMessage());
+ }
+ return httpProcessorConfiguration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8ccf9f6/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
index d76d839..dec9d03 100644
--- a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
@@ -19,6 +19,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.data.util.ActivityUtil;
@@ -34,6 +35,7 @@ import javax.validation.ValidatorFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -59,6 +61,11 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
// //private String authHeader;
//
+ public SimpleHTTPGetProcessor() {
+ LOGGER.info("creating SimpleHTTPGetProcessor");
+ this.configuration = HttpConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("http"));
+ }
+
public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
LOGGER.info("creating SimpleHTTPGetProcessor");
LOGGER.info(processorConfiguration.toString());
@@ -137,9 +144,7 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
return result;
}
- HttpGet httpget = new HttpGet(uri);
- httpget.addHeader("content-type", this.configuration.getContentType());
- //httpget.addHeader("Authorization", String.format("Basic %s", authHeader));
+ HttpGet httpget = prepareHttpGet(uri);
CloseableHttpResponse response = null;
@@ -184,6 +189,12 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
}
+ public HttpGet prepareHttpGet(URI uri) {
+ HttpGet httpget = new HttpGet(uri);
+ httpget.addHeader("content-type", this.configuration.getContentType());
+ return httpget;
+ }
+
@Override
public void prepare(Object configurationObject) {
@@ -198,12 +209,6 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
.setPath(this.configuration.getResourceUri());
httpclient = HttpClients.createDefault();
- // StringBuilder stringBuilder = new StringBuilder();
-// stringBuilder.append(peoplePatternConfiguration.getUsername());
-// stringBuilder.append(":");
-// stringBuilder.append(peoplePatternConfiguration.getPassword());
-// String string = stringBuilder.toString();
-// authHeader = Base64.encodeBase64String(string.getBytes());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8ccf9f6/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
index 77965c4..438937f 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
@@ -5,6 +5,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import org.apache.streams.components.http.HttpProcessorConfiguration;
import org.apache.streams.components.http.SimpleHTTPGetProcessor;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.pojo.json.Activity;
@@ -17,6 +18,13 @@ import java.util.Map;
*/
public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements StreamsProcessor {
+ public TwitterUrlApiProcessor() {
+ super();
+ this.configuration.setHostname("urls.api.twitter.com");
+ this.configuration.setResourceUri("/1/urls/count.json");
+ this.configuration.setExtension("twitter_url_count");
+ }
+
public TwitterUrlApiProcessor(HttpProcessorConfiguration processorConfiguration) {
super(processorConfiguration);
this.configuration.setHostname("urls.api.twitter.com");
[18/32] incubator-streams git commit: STREAMS-204 | Added in a Google
Plus Deserializer and Processor for Comments. The processor takes any
comments found for the passed in Activity (via a to-be-seen provider) and
appends them as ActivityObjects. A total
Posted by sb...@apache.org.
STREAMS-204 | Added in a Google Plus Deserializer and Processor for Comments. The processor takes any comments found for the passed in Activity (via a to-be-seen provider) and appends them as ActivityObjects. A total "comment_count" attribute is also stored in the "extensions" section of the Activity.
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/08e21453
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/08e21453
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/08e21453
Branch: refs/heads/STREAMS-212
Commit: 08e21453084e79661f1687a47f1310ec44568eaa
Parents: 2358b4e
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Wed Oct 29 17:44:39 2014 -0500
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Thu Nov 6 14:04:27 2014 -0600
----------------------------------------------------------------------
.../processor/GooglePlusCommentProcessor.java | 88 +++++++++++++-
.../util/GPlusCommentDeserializer.java | 99 +++++++++++++++-
.../serializer/util/GooglePlusActivityUtil.java | 50 ++++++++
.../gplus/GooglePLusCommentSerDeTest.java | 115 ++++++++++++++++++-
.../resources/google_plus_comments_jsons.txt | 3 +
5 files changed, 343 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
index 7dc4d56..583c741 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
@@ -1,7 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus.processor;
-/**
- * Created by rdouglas on 10/29/14.
- */
-public class GooglePlusCommentProcessor {
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Comment;
+import com.google.api.services.plus.model.Person;
+import com.google.gplus.serializer.util.GPlusActivityDeserializer;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.pojo.json.Activity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class GooglePlusCommentProcessor implements StreamsProcessor {
+ private final static String STREAMS_ID = "GooglePlusCommentProcessor";
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentProcessor.class);
+ private GooglePlusActivityUtil googlePlusActivityUtil;
+ private int count;
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ StreamsDatum result = null;
+
+ try {
+ Object item = entry.getDocument();
+ LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
+
+ //Get G+ activity ID from our own activity ID
+ if (item instanceof Activity) {
+ Activity activity = (Activity) item;
+ String activityId = getGPlusID(activity.getId());
+
+ //Call Google Plus API to get list of comments for this activity ID
+ /**TODO: FILL ME OUT WITH THE API CALL**/
+ List<Comment> comments = Lists.newArrayList();
+
+ googlePlusActivityUtil.updateActivity(comments, activity);
+ result = new StreamsDatum(activity);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.error("Exception while converting Comment to Activity: {}", e.getMessage());
+ }
+
+ if( result != null )
+ return com.google.common.collect.Lists.newArrayList(result);
+ else
+ return com.google.common.collect.Lists.newArrayList();
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ googlePlusActivityUtil = new GooglePlusActivityUtil();
+ count = 0;
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ private String getGPlusID(String activityID) {
+ String[] activityParts = activityID.split(":");
+ return (activityParts.length > 0) ? activityParts[activityParts.length - 1] : "";
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
index 51446e0..513bdc7 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
@@ -1,7 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus.serializer.util;
-/**
- * Created by rdouglas on 10/29/14.
- */
-public class GPlusCommentDeserializer {
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.client.util.DateTime;
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Comment;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+public class GPlusCommentDeserializer extends JsonDeserializer<Comment> {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GPlusActivityDeserializer.class);
+
+ /**
+ * Because the GooglePlus Comment object {@link com.google.api.services.plus.model.Comment} contains complex objects
+ * within its hierarchy, we have to use a custom deserializer
+ *
+ * @param jsonParser
+ * @param deserializationContext
+ * @return The deserialized {@link com.google.api.services.plus.model.Comment} object
+ * @throws java.io.IOException
+ * @throws com.fasterxml.jackson.core.JsonProcessingException
+ */
+ @Override
+ public Comment deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+
+ JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+ ObjectMapper objectMapper = new StreamsJacksonMapper();
+ Comment comment = new Comment();
+
+ try {
+ comment.setEtag(node.get("etag").asText());
+ comment.setVerb(node.get("verb").asText());
+ comment.setId(node.get("id").asText());
+ comment.setPublished(DateTime.parseRfc3339(node.get("published").asText()));
+ comment.setUpdated(DateTime.parseRfc3339(node.get("updated").asText()));
+
+ Comment.Actor actor = new Comment.Actor();
+ JsonNode actorNode = node.get("actor");
+ actor.setDisplayName(actorNode.get("displayName").asText());
+ actor.setUrl(actorNode.get("url").asText());
+
+ Comment.Actor.Image image = new Comment.Actor.Image();
+ JsonNode imageNode = actorNode.get("image");
+ image.setUrl(imageNode.get("url").asText());
+
+ actor.setImage(image);
+
+ comment.setObject(objectMapper.readValue(objectMapper.writeValueAsString(node.get("object")), Comment.PlusObject.class));
+
+ comment.setSelfLink(node.get("selfLink").asText());
+
+ List<Comment.InReplyTo> replies = Lists.newArrayList();
+ for(JsonNode reply : node.get("inReplyTo")) {
+ Comment.InReplyTo r = objectMapper.readValue(objectMapper.writeValueAsString(reply), Comment.InReplyTo.class);
+ replies.add(r);
+ }
+
+ comment.setInReplyTo(replies);
+
+ Comment.Plusoners plusoners = new Comment.Plusoners();
+ JsonNode plusonersNode = node.get("plusoners");
+ plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
+ comment.setPlusoners(plusoners);
+ } catch (Exception e) {
+ LOGGER.error("Exception while trying to deserialize activity object: {}", e);
+ }
+
+ return comment;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
index 73b4b0d..4e330fa 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
@@ -19,6 +19,8 @@
package com.google.gplus.serializer.util;
+import com.google.api.client.util.Maps;
+import com.google.api.services.plus.model.Comment;
import com.google.api.services.plus.model.Person;
import org.apache.streams.pojo.json.*;
import org.apache.streams.pojo.json.Activity;
@@ -31,6 +33,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
+import java.util.List;
import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
@@ -58,6 +61,22 @@ public class GooglePlusActivityUtil {
}
/**
+ * Given a {@link List} of {@link com.google.api.services.plus.model.Comment} objects and an
+ * {@link org.apache.streams.pojo.json.Activity}, update that Activity to contain all comments
+ *
+ * @param comments
+ * @param activity
+ */
+ public static void updateActivity(List<Comment> comments, Activity activity) {
+ for(Comment comment : comments) {
+ addComment(activity, comment);
+ }
+
+ Map<String, Object> extensions = ensureExtensions(activity);
+ extensions.put("comment_count", comments.size());
+ }
+
+ /**
* Given a Google Plus {@link com.google.api.services.plus.model.Activity},
* convert that into an Activity streams formatted {@link org.apache.streams.pojo.json.Activity}
*
@@ -88,6 +107,37 @@ public class GooglePlusActivityUtil {
}
/**
+ * Adds a single {@link com.google.api.services.plus.model.Comment} to the Object.Attachments
+ * section of the passed in {@link org.apache.streams.pojo.json.Activity}
+ *
+ * @param activity
+ * @param comment
+ */
+ private static void addComment(Activity activity, Comment comment) {
+ ActivityObject obj = new ActivityObject();
+
+ obj.setId(comment.getId());
+ obj.setPublished(new DateTime(String.valueOf(comment.getPublished())));
+ obj.setUpdated(new DateTime(String.valueOf(comment.getUpdated())));
+ obj.setContent(comment.getObject().getContent());
+ obj.setObjectType(comment.getObject().getObjectType());
+
+ Map<String, Object> extensions = Maps.newHashMap();
+ extensions.put("googlePlus", comment);
+
+ obj.setAdditionalProperty("extensions", extensions);
+
+ if(activity.getObject() == null) {
+ activity.setObject(new ActivityObject());
+ }
+ if(activity.getObject().getAttachments() == null) {
+ activity.getObject().setAttachments(new ArrayList<ActivityObject>());
+ }
+
+ activity.getObject().getAttachments().add(obj);
+ }
+
+ /**
* Add in necessary extensions from the passed in {@link com.google.api.services.plus.model.Activity} to the
* {@link org.apache.streams.pojo.json.Activity} object
*
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
index b3479f9..9fea22c 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
@@ -1,7 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package com.google.gplus;
-/**
- * Created by rdouglas on 10/29/14.
- */
-public class GooglePLusCommentSerDeTest {
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.client.util.Lists;
+import com.google.api.services.plus.model.Comment;
+import com.google.gplus.serializer.util.GPlusCommentDeserializer;
+import com.google.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class GooglePlusCommentSerDeTest {
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentSerDeTest.class);
+ private ObjectMapper objectMapper;
+ private GooglePlusActivityUtil googlePlusActivityUtil;
+
+ @Before
+ public void setup() {
+ objectMapper = new StreamsJacksonMapper();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Comment.class, new GPlusCommentDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ googlePlusActivityUtil = new GooglePlusActivityUtil();
+ }
+
+ @org.junit.Test
+ public void testCommentObjects() {
+ InputStream is = GooglePlusPersonSerDeTest.class.getResourceAsStream("/google_plus_comments_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ Activity activity = new Activity();
+ List<Comment> comments = Lists.newArrayList();
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Comment comment = objectMapper.readValue(line, Comment.class);
+
+ LOGGER.info("comment: {}", comment);
+
+ assertNotNull(comment);
+ assertNotNull(comment.getEtag());
+ assertNotNull(comment.getId());
+ assertNotNull(comment.getInReplyTo());
+ assertNotNull(comment.getObject());
+ assertNotNull(comment.getPlusoners());
+ assertNotNull(comment.getPublished());
+ assertNotNull(comment.getUpdated());
+ assertNotNull(comment.getSelfLink());
+ assertEquals(comment.getVerb(), "post");
+
+ comments.add(comment);
+ }
+ }
+
+ assertEquals(comments.size(), 3);
+
+ googlePlusActivityUtil.updateActivity(comments, activity);
+ assertNotNull(activity);
+ assertNotNull(activity.getObject());
+ assertEquals(activity.getObject().getAttachments().size(), 3);
+ } catch (Exception e) {
+ LOGGER.error("Exception while testing serializability: {}", e);
+ }
+ }
+
+ @org.junit.Test
+ public void testEmptyComments() {
+ Activity activity = new Activity();
+
+ googlePlusActivityUtil.updateActivity(new ArrayList<Comment>(), activity);
+
+ assertNull(activity.getObject());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/08e21453/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
index e69de29..e52abf1 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/google_plus_comments_jsons.txt
@@ -0,0 +1,3 @@
+{ "kind": "plus#comment", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/ANtsF6kh7Ztc8-ufvOzWL234qfY\"", "verb": "post", "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k.1414517802192122", "published": "2014-10-28T17:36:42.192Z", "updated": "2014-10-28T17:36:42.192Z", "actor": { "id": "110922531601810522565", "displayName": "Adalberto Hernandez", "url": "https://plus.google.com/110922531601810522565", "image": { "url": "https://lh4.googleusercontent.com/-VpIxk0xr8i0/AAAAAAAAAAI/AAAAAAAADl4/HgEI76iIDPk/photo.jpg?sz=50" } }, "object": { "objectType": "comment", "content": "I would buy the nexus 6 I still think the screen size is perfect for me" }, "selfLink": "https://content.googleapis.com/plus/v1/comments/z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k#1414517802192122", "inReplyTo": [ { "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k", "url": "https://plus.google.com/101127444819996140229/posts/QXuZzrgguaK" } ], "plusoners": { "totalItems": 6 } }
+{ "kind": "plus#comment", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/81qALbxT1WteuRVngMKhNsvF2hM\"", "verb": "post", "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k.1414517816125433", "published": "2014-10-28T17:36:56.125Z", "updated": "2014-10-28T17:38:14.000Z", "actor": { "id": "107131626164728505473", "displayName": "Vance McAlister", "url": "https://plus.google.com/107131626164728505473", "image": { "url": "https://lh4.googleusercontent.com/-ENY-GPKzlpM/AAAAAAAAAAI/AAAAAAAAQPs/qTzDXHiDs6c/photo.jpg?sz=50" } }, "object": { "objectType": "comment", "content": "Only if they have gotten rid of Blur and it is as much pure Android as the Moto X." }, "selfLink": "https://content.googleapis.com/plus/v1/comments/z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k#1414517816125433", "inReplyTo": [ { "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k", "url": "https://plus.google.com/101127444819996140229/posts/QXuZzrgguaK" } ], "plusoners": { "totalItems": 7 } }, { "kind": "plus#comment", "etag": "\"Vea_b94Y77GDGgR
K7gFNPnolKQw/3tvcbp0tAYDvMPPViM-Iq8CsVLU\"", "verb": "post", "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k.1414517894284415", "published": "2014-10-28T17:38:14.284Z", "updated": "2014-10-28T17:38:14.284Z", "actor": { "id": "116526038399605835276", "displayName": "Thiago Vinhas", "url": "https://plus.google.com/116526038399605835276", "image": { "url": "https://lh6.googleusercontent.com/-ze6E8Zj8LM8/AAAAAAAAAAI/AAAAAAAAcII/2htWB7qbHs4/photo.jpg?sz=50" } }, "object": { "objectType": "comment", "content": "Verizon? No, thank you. I prefer to be kicked on the balls." }, "selfLink": "https://content.googleapis.com/plus/v1/comments/z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k#1414517894284415", "inReplyTo": [ { "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k", "url": "https://plus.google.com/101127444819996140229/posts/QXuZzrgguaK" } ], "plusoners": { "totalItems": 52 } }
+{ "kind": "plus#comment", "etag": "\"Vea_b94Y77GDGgRK7gFNPnolKQw/DCSYhsbyQnRvdVQ8U7e1B6Rw9F8\"", "verb": "post", "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k.1414518523966061", "published": "2014-10-28T17:48:43.966Z", "updated": "2014-10-28T17:48:43.966Z", "actor": { "id": "115669911467014111133", "displayName": "Eduardo Castaneda", "url": "https://plus.google.com/115669911467014111133", "image": { "url": "https://lh4.googleusercontent.com/-I0ghHQLUDC4/AAAAAAAAAAI/AAAAAAAABFk/YkeCQCLzwk8/photo.jpg?sz=50" } }, "object": { "objectType": "comment", "content": "how the fuck is this not the new nexus?!?! " }, "selfLink": "https://content.googleapis.com/plus/v1/comments/z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k#1414518523966061", "inReplyTo": [ { "id": "z132c114nq3eijq4g04cdlo4yo33x3u5zvg0k", "url": "https://plus.google.com/101127444819996140229/posts/QXuZzrgguaK" } ], "plusoners": { "totalItems": 18 } }
\ No newline at end of file
[19/32] incubator-streams git commit: STREAMS-204 | Removed file with
typo in name
Posted by sb...@apache.org.
STREAMS-204 | Removed file with typo in name
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e792bb9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e792bb9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e792bb9e
Branch: refs/heads/STREAMS-212
Commit: e792bb9e1ef4d9c9257acd6e97bca466511f6d0e
Parents: 08e2145
Author: Robert Douglas <rd...@w2ogroup.com>
Authored: Thu Nov 6 14:10:17 2014 -0600
Committer: Robert Douglas <rd...@w2ogroup.com>
Committed: Thu Nov 6 14:12:03 2014 -0600
----------------------------------------------------------------------
.../gplus/GooglePLusCommentSerDeTest.java | 114 -------------------
1 file changed, 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e792bb9e/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
deleted file mode 100644
index 9fea22c..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePLusCommentSerDeTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.client.util.Lists;
-import com.google.api.services.plus.model.Comment;
-import com.google.gplus.serializer.util.GPlusCommentDeserializer;
-import com.google.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.commons.lang.StringUtils;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
-public class GooglePlusCommentSerDeTest {
- private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentSerDeTest.class);
- private ObjectMapper objectMapper;
- private GooglePlusActivityUtil googlePlusActivityUtil;
-
- @Before
- public void setup() {
- objectMapper = new StreamsJacksonMapper();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Comment.class, new GPlusCommentDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- googlePlusActivityUtil = new GooglePlusActivityUtil();
- }
-
- @org.junit.Test
- public void testCommentObjects() {
- InputStream is = GooglePlusPersonSerDeTest.class.getResourceAsStream("/google_plus_comments_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- Activity activity = new Activity();
- List<Comment> comments = Lists.newArrayList();
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Comment comment = objectMapper.readValue(line, Comment.class);
-
- LOGGER.info("comment: {}", comment);
-
- assertNotNull(comment);
- assertNotNull(comment.getEtag());
- assertNotNull(comment.getId());
- assertNotNull(comment.getInReplyTo());
- assertNotNull(comment.getObject());
- assertNotNull(comment.getPlusoners());
- assertNotNull(comment.getPublished());
- assertNotNull(comment.getUpdated());
- assertNotNull(comment.getSelfLink());
- assertEquals(comment.getVerb(), "post");
-
- comments.add(comment);
- }
- }
-
- assertEquals(comments.size(), 3);
-
- googlePlusActivityUtil.updateActivity(comments, activity);
- assertNotNull(activity);
- assertNotNull(activity.getObject());
- assertEquals(activity.getObject().getAttachments().size(), 3);
- } catch (Exception e) {
- LOGGER.error("Exception while testing serializability: {}", e);
- }
- }
-
- @org.junit.Test
- public void testEmptyComments() {
- Activity activity = new Activity();
-
- googlePlusActivityUtil.updateActivity(new ArrayList<Comment>(), activity);
-
- assertNull(activity.getObject());
- }
-}
[11/32] incubator-streams git commit: Fixed tests by unregistering
previously registered mbeans
Posted by sb...@apache.org.
Fixed tests by unregistering previously registered mbeans
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d71568dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d71568dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d71568dc
Branch: refs/heads/STREAMS-212
Commit: d71568dc72716d8d6b10ef0de74df5325f3d1336
Parents: 7e65a42
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Mon Oct 20 15:30:28 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Mon Oct 20 15:30:28 2014 -0500
----------------------------------------------------------------------
.../local/builders/LocalStreamBuilder.java | 5 +++
.../local/builders/LocalStreamBuilderTest.java | 32 +++++++++++++++++---
2 files changed, 32 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d71568dc/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 57f3aa4..2161638 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -20,6 +20,7 @@ package org.apache.streams.local.builders;
import org.apache.log4j.spi.LoggerFactory;
import org.apache.streams.core.*;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.streams.local.executors.ShutdownStreamOnUnhandleThrowableThreadPoolExecutor;
import org.apache.streams.local.queues.ThroughputQueue;
import org.apache.streams.local.tasks.LocalStreamProcessMonitorThread;
@@ -271,6 +272,8 @@ public class LocalStreamBuilder implements StreamBuilder {
for(StreamComponent prov : this.providers.values()) {
StreamsTask task = prov.createConnectedTask(getTimeout());
task.setStreamConfig(this.streamConfig);
+ StreamsTaskCounter counter = new StreamsTaskCounter(prov.getId());
+ task.setStreamsTaskCounter(counter);
this.executor.submit(task);
provTasks.put(prov.getId(), (StreamsProviderTask) task);
if( prov.isOperationCountable() ) {
@@ -284,8 +287,10 @@ public class LocalStreamBuilder implements StreamBuilder {
for(StreamComponent comp : this.components.values()) {
int tasks = comp.getNumTasks();
List<StreamsTask> compTasks = new LinkedList<StreamsTask>();
+ StreamsTaskCounter counter = new StreamsTaskCounter(comp.getId());
for(int i=0; i < tasks; ++i) {
StreamsTask task = comp.createConnectedTask(getTimeout());
+ task.setStreamsTaskCounter(counter);
task.setStreamConfig(this.streamConfig);
this.futures.put(task, this.executor.submit(task));
compTasks.add(task);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d71568dc/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
index e602181..fea7e53 100644
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
@@ -43,6 +43,7 @@ import org.apache.streams.core.StreamBuilder;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsPersistWriter;
import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.streams.local.queues.ThroughputQueue;
import org.apache.streams.local.test.processors.PassthroughDatumCounterProcessor;
import org.apache.streams.local.test.processors.SlowProcessor;
@@ -81,10 +82,17 @@ public class LocalStreamBuilderTest extends RandomizedTest {
} catch (MalformedObjectNameException|InstanceNotFoundException|MBeanRegistrationException e) {
//No-op
}
+ try {
+ mbs.unregisterMBean(new ObjectName((String.format(StreamsTaskCounter.NAME_TEMPLATE, id))));
+ } catch (MalformedObjectNameException|InstanceNotFoundException|MBeanRegistrationException e) {
+ //No-op
+ }
}
}
+
+
@Test
public void testStreamIdValidations() {
StreamBuilder builder = new LocalStreamBuilder();
@@ -104,7 +112,7 @@ public class LocalStreamBuilderTest extends RandomizedTest {
exp = e;
}
assertNotNull(exp);
- removeRegisteredMBeans("1", "2");
+ removeRegisteredMBeans("1", "2", "id");
}
@Test
@@ -160,9 +168,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
}
} finally {
for(int i=0; i < numProcessors; ++i) {
- removeRegisteredMBeans(processorId+i);
+ removeRegisteredMBeans(processorId+i, processorId+i+"-"+PassthroughDatumCounterProcessor.class.getCanonicalName());
}
- removeRegisteredMBeans("writer");
+ removeRegisteredMBeans("writer", "numeric_provider");
}
}
@@ -199,7 +207,7 @@ public class LocalStreamBuilderTest extends RandomizedTest {
for(int i=0; i < numProcessors; ++i) {
removeRegisteredMBeans(processorId+i);
}
- removeRegisteredMBeans("writer");
+ removeRegisteredMBeans("writer", "numeric_provider");
}
}
@@ -221,7 +229,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
assertEquals(numDatums2, PassthroughDatumCounterProcessor.COUNTS.get("proc2").get());
assertEquals(numDatums1+numDatums2, DatumCounterWriter.COUNTS.get("writer").get());
} finally {
- removeRegisteredMBeans("proc1", "proc2", "writer1");
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
+ removeRegisteredMBeans("proc1", "proc2", "writer1", "sp1", "sp2");
}
}
@@ -239,6 +249,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
assertEquals(numDatums, PassthroughDatumCounterProcessor.COUNTS.get("proc2").get());
assertEquals(numDatums*2, DatumCounterWriter.COUNTS.get("writer").get());
} finally {
+ String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
removeRegisteredMBeans("prov1", "proc1", "proc2", "w1");
}
}
@@ -257,6 +270,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
builder.start();
assertEquals(numDatums, DatumCounterWriter.COUNTS.get("writer").get());
} finally {
+ String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
removeRegisteredMBeans("prov1", "proc1", "w1");
}
}
@@ -278,6 +294,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
//We care mostly that it doesn't terminate too early. With thread shutdowns, etc, the actual time is indeterminate. Just make sure there is an upper bound
assertThat((int) (end - start), is(allOf(greaterThanOrEqualTo(timeout), lessThanOrEqualTo(4 * timeout))));
} finally {
+ String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
removeRegisteredMBeans("prov1", "proc1", "proc2", "w1");
}
}
@@ -305,6 +324,9 @@ public class LocalStreamBuilderTest extends RandomizedTest {
service.awaitTermination(30000, TimeUnit.MILLISECONDS);
assertThat(Thread.activeCount(), is(equalTo(before)));
} finally {
+ String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
+ String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
+ String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
removeRegisteredMBeans("prov1", "proc1", "w1");
}
}
[29/32] incubator-streams git commit: Merge remote-tracking branch
'rbnks/gplus'
Posted by sb...@apache.org.
Merge remote-tracking branch 'rbnks/gplus'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/bfa9466b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/bfa9466b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/bfa9466b
Branch: refs/heads/STREAMS-212
Commit: bfa9466b024db1b178172deb38a91e0e0ce05a20
Parents: 01716fe 4d6d02a
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Nov 7 14:47:42 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Nov 7 14:47:42 2014 -0600
----------------------------------------------------------------------
.../google-gplus/pom.xml | 32 +++
.../gplus/provider/AbstractGPlusProvider.java | 234 ++++++++++++++++
.../gplus/provider/GPlusActivitySerializer.java | 4 +-
.../gplus/provider/GPlusDataCollector.java | 50 ++++
.../provider/GPlusHistoryProviderTask.java | 106 --------
.../google/gplus/provider/GPlusProvider.java | 189 -------------
.../provider/GPlusUserActivityCollector.java | 108 ++++++++
.../provider/GPlusUserActivityProvider.java | 18 ++
.../gplus/provider/GPlusUserDataCollector.java | 79 ++++++
.../gplus/provider/GPlusUserDataProvider.java | 18 ++
.../com/google/gplus/GPlusConfiguration.json | 44 ++-
.../provider/TestAbstractGPlusProvider.java | 82 ++++++
.../TestGPlusUserActivityCollector.java | 268 +++++++++++++++++++
.../provider/TestGPlusUserDataCollector.java | 131 +++++++++
streams-contrib/streams-provider-google/pom.xml | 6 +
.../backoff/AbstractBackOffStrategy.java | 15 +-
16 files changed, 1080 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
[02/32] incubator-streams git commit: added streams-components module
added streams-processor-http module activated streams-pojo-extensions module
tweaks to pojo, require id and remove non-sensical defaults
Posted by sb...@apache.org.
added streams-components module
added streams-processor-http module
activated streams-pojo-extensions module
tweaks to pojo, require id and remove non-sensical defaults
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/34232ad8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/34232ad8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/34232ad8
Branch: refs/heads/STREAMS-212
Commit: 34232ad87b50253913fba0dabb6fa9af591f388d
Parents: 1464819
Author: sblackmon <sb...@apache.org>
Authored: Mon Sep 15 13:50:32 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Mon Sep 15 13:50:32 2014 -0500
----------------------------------------------------------------------
pom.xml | 2 +
.../streams-processor-http/README.md | 16 ++
.../streams-processor-http/pom.xml | 42 +++-
.../components/http/SimpleHTTPGetProcessor.java | 213 +++++++++++++++++++
.../HttpProcessorConfiguration.json | 47 ++++
.../api/FacebookPostActivitySerializer.java | 1 -
.../streams-provider-twitter/pom.xml | 5 +
.../processor/TwitterUrlApiProcessor.java | 44 ++++
streams-pojo-extensions/pom.xml | 64 ++++++
.../apache/streams/data/util/ExtensionUtil.java | 94 ++++++++
.../apache/streams/data/util/ActivityUtil.java | 14 +-
.../org/apache/streams/pojo/json/activity.json | 3 +-
.../org/apache/streams/pojo/json/object.json | 2 +-
13 files changed, 537 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c2e1766..59fa634 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,7 +96,9 @@
<module>streams-osgi-components</module>
<module>streams-core</module>
<module>streams-config</module>
+ <module>streams-components</module>
<module>streams-pojo</module>
+ <module>streams-pojo-extensions</module>
<module>streams-util</module>
<module>streams-contrib</module>
<module>streams-runtimes</module>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-components/streams-processor-http/README.md
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/README.md b/streams-components/streams-processor-http/README.md
new file mode 100644
index 0000000..62dd4c1
--- /dev/null
+++ b/streams-components/streams-processor-http/README.md
@@ -0,0 +1,16 @@
+streams-processor-http
+=====================
+
+Hit an http endpoint and place the result in extensions
+
+Example SimpleHTTPGetProcessor configuration:
+
+ "http": {
+ "protocol": "http",
+ "hostname": "urls.api.twitter.com",
+ "port": 9300,
+ "resourceUri": "1/urls/count.json"
+ }
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-components/streams-processor-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/pom.xml b/streams-components/streams-processor-http/pom.xml
index 67538aa..d9215ad 100644
--- a/streams-components/streams-processor-http/pom.xml
+++ b/streams-components/streams-processor-http/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-project</artifactId>
+ <artifactId>streams-components</artifactId>
<version>0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -40,6 +40,45 @@
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo-extensions</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.3.5</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
</dependencies>
<build>
@@ -100,6 +139,7 @@
<targetPackage>org.apache.streams.http</targetPackage>
<useLongIntegers>true</useLongIntegers>
<useJodaDates>true</useJodaDates>
+ <includeJsr303Annotations>true</includeJsr303Annotations>
</configuration>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
new file mode 100644
index 0000000..d76d839
--- /dev/null
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
@@ -0,0 +1,213 @@
+package org.apache.streams.components.http;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigValue;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.data.util.ActivityUtil;
+import org.apache.streams.data.util.ExtensionUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.Validation;
+import javax.validation.ValidatorFactory;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
+
+ private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
+
+ // from root config id
+ private final static String EXTENSION = "account_type";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPGetProcessor.class);
+
+ protected ObjectMapper mapper;
+
+ protected URIBuilder uriBuilder;
+
+ protected CloseableHttpClient httpclient;
+
+ protected HttpProcessorConfiguration configuration;
+//
+// // authorized only
+// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
+// //private String authHeader;
+//
+ public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
+ LOGGER.info("creating SimpleHTTPGetProcessor");
+ LOGGER.info(processorConfiguration.toString());
+ this.configuration = processorConfiguration;
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+
+ return Maps.newHashMap();
+ }
+
+ /**
+ Override this to store a result other than exact json representation of response
+ */
+ protected ObjectNode prepareExtensionFragment(String entityString) {
+
+ try {
+ return mapper.readValue(entityString, ObjectNode.class);
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ }
+ }
+
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode getRootDocument(StreamsDatum datum) {
+
+ try {
+ String json = datum.getDocument() instanceof String ?
+ (String) datum.getDocument() :
+ mapper.writeValueAsString(datum.getDocument());
+ return mapper.readValue(json, ObjectNode.class);
+ } catch (JsonProcessingException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ }
+
+ }
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode getEntityToExtend(ObjectNode rootDocument) {
+
+ if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
+ return rootDocument;
+ else
+ return (ObjectNode) rootDocument.get(this.configuration.getEntity().toString());
+
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+
+ List<StreamsDatum> result = Lists.newArrayList();
+
+ ObjectNode rootDocument = getRootDocument(entry);
+
+ Map<String, String> params = prepareParams(entry);
+
+ URI uri;
+ for( Map.Entry<String,String> param : params.entrySet()) {
+ uriBuilder = uriBuilder.setParameter(param.getKey(), param.getValue());
+ }
+ try {
+ uri = uriBuilder.build();
+ } catch (URISyntaxException e) {
+ LOGGER.error("URI error {}", uriBuilder.toString());
+ return result;
+ }
+
+ HttpGet httpget = new HttpGet(uri);
+ httpget.addHeader("content-type", this.configuration.getContentType());
+ //httpget.addHeader("Authorization", String.format("Basic %s", authHeader));
+
+ CloseableHttpResponse response = null;
+
+ String entityString = null;
+ try {
+ response = httpclient.execute(httpget);
+ HttpEntity entity = response.getEntity();
+ // TODO: handle rate-limiting
+ if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
+ entityString = EntityUtils.toString(entity);
+ }
+ } catch (IOException e) {
+ LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, e.getMessage());
+ return result;
+ } finally {
+ try {
+ response.close();
+ } catch (IOException e) {}
+ try {
+ httpclient.close();
+ } catch (IOException e) {}
+ }
+
+ if( entityString == null )
+ return result;
+
+ LOGGER.debug(entityString);
+
+ ObjectNode extensionFragment = prepareExtensionFragment(entityString);
+
+ ObjectNode extensionEntity = getEntityToExtend(rootDocument);
+
+ ExtensionUtil.ensureExtensions(extensionEntity);
+
+ ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
+
+ entry.setDocument(extensionEntity);
+
+ result.add(entry);
+
+ return result;
+
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
+ Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
+
+ mapper = StreamsJacksonMapper.getInstance();
+
+ uriBuilder = new URIBuilder()
+ .setScheme(this.configuration.getProtocol())
+ .setHost(this.configuration.getHostname())
+ .setPath(this.configuration.getResourceUri());
+
+ httpclient = HttpClients.createDefault();
+ // StringBuilder stringBuilder = new StringBuilder();
+// stringBuilder.append(peoplePatternConfiguration.getUsername());
+// stringBuilder.append(":");
+// stringBuilder.append(peoplePatternConfiguration.getPassword());
+// String string = stringBuilder.toString();
+// authHeader = Base64.encodeBase64String(string.getBytes());
+ }
+
+ @Override
+ public void cleanUp() {
+ LOGGER.info("shutting down SimpleHTTPGetProcessor");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json b/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
new file mode 100644
index 0000000..40c3bcd
--- /dev/null
+++ b/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
@@ -0,0 +1,47 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.components.http.HttpProcessorConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "protocol": {
+ "type": "string",
+ "description": "Protocol",
+ "default": "http"
+ },
+ "hostname": {
+ "type": "string",
+ "description": "Hostname",
+ "required" : true
+ },
+ "port": {
+ "type": "integer",
+ "description": "Port",
+ "default": 80
+ },
+ "resourceUri": {
+ "type": "string",
+ "description": "Resource URI",
+ "required" : true
+ },
+ "content-type": {
+ "type": "string",
+ "description": "Resource URI",
+ "required" : true,
+ "default": "application/json"
+ },
+ "entity": {
+ "type": "string",
+ "description": "Entity to extend",
+ "enum": [ "activity", "actor", "object", "target" ],
+ "required" : true,
+ "default": "activity"
+ },
+ "extension": {
+ "type": "string",
+ "description": "Extension identifier",
+ "required" : true
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
index aa718fb..de39262 100644
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
+++ b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
@@ -78,7 +78,6 @@ public class FacebookPostActivitySerializer implements ActivitySerializer<org.ap
setProvider(activity);
setObjectType(post.getType(), activity);
parseObject(activity, mapper.convertValue(post, ObjectNode.class));
- fixObjectId(activity);
fixContentFromSummary(activity);
activity.setVerb("post");
List<String> links = Lists.newLinkedList();
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
index 79d1608..f0d65f8 100644
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@ -54,6 +54,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-processor-http</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
new file mode 100644
index 0000000..77965c4
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
@@ -0,0 +1,44 @@
+package org.apache.streams.twitter.processor;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import org.apache.streams.components.http.HttpProcessorConfiguration;
+import org.apache.streams.components.http.SimpleHTTPGetProcessor;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.pojo.json.Activity;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by sblackmon on 9/14/14.
+ */
+public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements StreamsProcessor {
+
+ public TwitterUrlApiProcessor(HttpProcessorConfiguration processorConfiguration) {
+ super(processorConfiguration);
+ this.configuration.setHostname("urls.api.twitter.com");
+ this.configuration.setResourceUri("/1/urls/count.json");
+ this.configuration.setExtension("twitter_url_count");
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ Preconditions.checkArgument(entry.getDocument() instanceof Activity);
+ Activity activity = mapper.convertValue(entry, Activity.class);
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(activity.getUrl()));
+ return super.process(entry);
+ }
+
+ @Override
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+
+ Map<String, String> params = Maps.newHashMap();
+
+ params.put("url", mapper.convertValue(entry, Activity.class).getUrl());
+
+ return params;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo-extensions/pom.xml
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/pom.xml b/streams-pojo-extensions/pom.xml
new file mode 100644
index 0000000..7f3f1a5
--- /dev/null
+++ b/streams-pojo-extensions/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-project</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>streams-pojo-extensions</artifactId>
+
+ <name>streams-pojo-extensions</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
new file mode 100644
index 0000000..a8d068a
--- /dev/null
+++ b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
@@ -0,0 +1,94 @@
+package org.apache.streams.data.util;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Maps;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.ActivityObject;
+
+import java.util.Map;
+
+public class ExtensionUtil {
+
+ /**
+ * Property on the activity object to use for extensions
+ */
+ public static final String EXTENSION_PROPERTY = "extensions";
+ /**
+ * The number of +1, Like, favorites, etc that the post has received
+ */
+ public static final String LIKES_EXTENSION = "likes";
+ /**
+ * The number of retweets, shares, etc that the post has received
+ */
+ public static final String REBROADCAST_EXTENSION = "rebroadcasts";
+ /**
+ * The language of the post
+ */
+ public static final String LANGUAGE_EXTENSION = "language";
+ /**
+ * Location that the post was made or the actor's residence
+ */
+ public static final String LOCATION_EXTENSION = "location";
+ /**
+ * Country that the post was made
+ */
+ public static final String LOCATION_EXTENSION_COUNTRY = "country";
+ /**
+ * Specific JSON-geo coordinates (long,lat)
+ */
+ public static final String LOCATION_EXTENSION_COORDINATES = "coordinates";
+
+ private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+
+ public static Map<String, Object> getExtensions(ObjectNode object) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ return extensions;
+ }
+
+ public static Object getExtension(ObjectNode object, String key) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ return extensions.get(key);
+ }
+
+ public static void setExtensions(ObjectNode object, Map<String, Object> extensions) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ activityObject.setAdditionalProperty(EXTENSION_PROPERTY, extensions);
+ };
+
+ public static void addExtension(ObjectNode object, String key, Object extension) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ extensions.put(key, extension);
+ };
+
+ public static void addExtensions(ObjectNode object, Map<String, Object> extensions) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ for( Map.Entry<String, Object> item : extensions.entrySet())
+ activityObject.getAdditionalProperties().put(item.getKey(), item.getValue());
+ };
+
+ public static void removeExtension(ObjectNode object, String key) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ extensions.remove(key);
+ };
+
+ /**
+ * Creates a standard extension property
+ * @param object objectnode to create the property in
+ * @return the Map representing the extensions property
+ */
+ @SuppressWarnings("unchecked")
+ public static Map<String, Object> ensureExtensions(ObjectNode object) {
+ ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ if(extensions == null) {
+ extensions = Maps.newHashMap();
+ setExtensions(object, extensions);
+ }
+ return getExtensions(object);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java b/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
index 3684b32..04ee923 100644
--- a/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
+++ b/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
@@ -19,6 +19,7 @@
package org.apache.streams.data.util;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.Activity;
@@ -61,7 +62,7 @@ public class ActivityUtil {
*/
public static final String LOCATION_EXTENSION_COORDINATES = "coordinates";
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+ private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
/**
* Creates a standard extension property
@@ -69,13 +70,14 @@ public class ActivityUtil {
* @return the Map representing the extensions property
*/
@SuppressWarnings("unchecked")
+ @Deprecated
public static Map<String, Object> ensureExtensions(Activity activity) {
- Map<String, Object> properties = (Map)activity.getAdditionalProperties().get(EXTENSION_PROPERTY);
- if(properties == null) {
- properties = new HashMap<String, Object>();
- activity.setAdditionalProperty(EXTENSION_PROPERTY, properties);
+ Map<String, Object> extensions = (Map)activity.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ if(extensions == null) {
+ extensions = new HashMap<String, Object>();
+ activity.setAdditionalProperty(EXTENSION_PROPERTY, extensions);
}
- return properties;
+ return extensions;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json b/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json
index 45c2276..a68ce00 100644
--- a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json
+++ b/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json
@@ -8,7 +8,8 @@
"properties": {
"id" :{
"type" : "string",
- "description" : "Uniquely identifies each activity within the service"
+ "description" : "Uniquely identifies each activity within the service",
+ "required" : true
},
"actor" : {
"type": "object",
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/34232ad8/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json b/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json
index d51db27..eec09a8 100644
--- a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json
+++ b/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/object.json
@@ -8,7 +8,7 @@
"id" : {
"type" : "string",
"description" : "Provides a permanent, universally unique identifier for the object in the form of an absolute IRI [RFC3987]. An object SHOULD contain a single id property. If an object does not contain an id property, consumers MAY use the value of the url property as a less-reliable, non-unique identifier.",
- "default" : "{link}"
+ "required" : true
},
"image" : {
"format":"image",
[16/32] incubator-streams git commit: Merge pull request #12 from
apache/master
Posted by sb...@apache.org.
Merge pull request #12 from apache/master
Merge Apache Master 2014/11/03
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/76763f89
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/76763f89
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/76763f89
Branch: refs/heads/STREAMS-212
Commit: 76763f89ea9432802af74e27a0085d914fa01af9
Parents: 5c2e832 9aebd0b
Author: Ryan Ebanks <rb...@users.noreply.github.com>
Authored: Mon Nov 3 10:41:01 2014 -0600
Committer: Ryan Ebanks <rb...@users.noreply.github.com>
Committed: Mon Nov 3 10:41:01 2014 -0600
----------------------------------------------------------------------
.../processor/PercolateTagProcessor.java | 10 +-
.../ElasticsearchWriterConfiguration.json | 10 ++
.../processor/GooglePlusTypeConverter.java | 5 +-
.../gplus/provider/GPlusActivitySerializer.java | 77 +--------
.../gplus/provider/GPlusEventProcessor.java | 3 -
.../util/GPlusActivityDeserializer.java | 162 +++++++++++++++++++
.../serializer/util/GooglePlusActivityUtil.java | 143 +++++++++++++++-
.../processor/GooglePlusActivitySerDeTest.java | 112 +++++++++++++
.../processor/GooglePlusTypeConverterTest.java | 63 +++++---
.../resources/google_plus_activity_jsons.txt | 5 +
10 files changed, 483 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
[22/32] incubator-streams git commit: Added some java docs
Posted by sb...@apache.org.
Added some java docs
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ff6cb52d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ff6cb52d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ff6cb52d
Branch: refs/heads/STREAMS-212
Commit: ff6cb52d90908d7be22c2d3439e00a682981e242
Parents: 4e0e9da
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Thu Nov 6 22:39:04 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Thu Nov 6 22:39:04 2014 -0600
----------------------------------------------------------------------
.../java/com/google/gplus/provider/AbstractGPlusProvider.java | 2 +-
.../com/google/gplus/provider/GPlusUserActivityCollector.java | 2 +-
.../java/com/google/gplus/provider/GPlusUserDataCollector.java | 2 +-
.../com/google/gplus/provider/TestAbstractGPlusProvider.java | 5 ++++-
4 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ff6cb52d/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
index daa34d5..6e59da2 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
@@ -52,7 +52,7 @@ import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
/**
- *
+ * Provider that creates a GPlus client and will run task that queue data to an outing queue
*/
public abstract class AbstractGPlusProvider implements StreamsProvider {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ff6cb52d/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
index c6952bf..d9a89d9 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
@@ -20,7 +20,7 @@ import org.slf4j.LoggerFactory;
import java.util.concurrent.BlockingQueue;
/**
- *
+ * Collects the public activities of a GPlus user. Has ability to filter by date ranges.
*/
public class GPlusUserActivityCollector implements Runnable {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ff6cb52d/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
index 6a5ce49..5269e10 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
@@ -17,7 +17,7 @@ import org.slf4j.LoggerFactory;
import java.util.concurrent.BlockingQueue;
/**
- *
+ * Collects user profile information for a specific GPlus user
*/
public class GPlusUserDataCollector implements Runnable{
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ff6cb52d/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
index 80b1b23..ced612b 100644
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
@@ -19,10 +19,13 @@ import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
/**
- *
+ * Unit tests for {@link com.google.gplus.provider.AbstractGPlusProvider}
*/
public class TestAbstractGPlusProvider extends RandomizedTest{
+ /**
+ * Test that every collector will be run and that data queued from the collectors will be processed.
+ */
@Test
@Repeat(iterations = 3)
public void testDataCollectorRunsPerUser() {
[14/32] incubator-streams git commit: Merged apache master
Posted by sb...@apache.org.
Merged apache master
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/377fb897
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/377fb897
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/377fb897
Branch: refs/heads/STREAMS-212
Commit: 377fb89717f34e6e109cf2655d6e3e335a494499
Parents: 015fade 5c2e832
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Oct 31 16:32:16 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Oct 31 16:32:16 2014 -0500
----------------------------------------------------------------------
streams-contrib/streams-amazon-aws/pom.xml | 2 +-
.../org/apache/streams/s3/S3Configurator.java | 55 ++++-----
.../org/apache/streams/s3/S3PersistReader.java | 11 +-
.../org/apache/streams/s3/S3PersistWriter.java | 10 +-
.../org/apache/streams/s3/S3Configuration.json | 18 ++-
.../streams-persist-elasticsearch/pom.xml | 47 +++++++
.../ElasticsearchMetadataUtil.java | 106 ++++++++++++++++
.../ElasticsearchPersistDeleter.java | 6 +-
.../ElasticsearchPersistUpdater.java | 6 +-
.../ElasticsearchPersistWriter.java | 47 +------
.../DatumFromMetadataAsDocumentProcessor.java | 122 +++++++++++++++++++
.../processor/DatumFromMetadataProcessor.java | 103 ++++++++++++++++
.../processor/DocumentToMetadataProcessor.java | 100 +++++++++++++++
.../ElasticsearchConfiguration.json | 3 +-
.../test/TestDatumFromMetadataProcessor.java | 81 ++++++++++++
.../test/TestDocumentToMetadataProcessor.java | 63 ++++++++++
.../test/TestElasticsearchPersistWriter.java | 70 +++++++++++
.../google-gplus/pom.xml | 2 +-
.../processor/GooglePlusTypeConverter.java | 91 ++++++++++++++
.../util/GPlusPersonDeserializer.java | 107 ++++++++++++++++
.../serializer/util/GooglePlusActivityUtil.java | 115 +++++++++++++++++
.../google/gplus/GooglePlusPersonSerDeTest.java | 97 +++++++++++++++
.../processor/GooglePlusTypeConverterTest.java | 98 +++++++++++++++
.../test/resources/google_plus_person_jsons.txt | 2 +
.../provider/TwitterTimelineProvider.java | 14 ++-
.../provider/TwitterTimelineProviderTest.java | 39 ++++++
streams-runtimes/streams-runtime-local/pom.xml | 6 +-
.../local/counters/DatumStatusCounter.java | 15 +--
.../local/counters/StreamsTaskCounter.java | 10 +-
.../local/tasks/StreamsProcessorTask.java | 3 +-
.../local/builders/LocalStreamBuilderTest.java | 12 ++
...nhandledThrowableThreadPoolExecutorTest.java | 11 ++
.../queues/ThroughputQueueMulitThreadTest.java | 10 ++
.../queues/ThroughputQueueSingleThreadTest.java | 11 ++
.../streams/local/tasks/BasicTasksTest.java | 15 +--
.../local/tasks/StreamsProviderTaskTest.java | 10 ++
.../tests/TestComponentsLocalStream.java | 13 +-
.../tests/TestExpectedDatumsPersitWriter.java | 13 +-
.../component/tests/TestFileReaderProvider.java | 12 +-
.../org/apache/streams/util/ComponentUtils.java | 33 +++++
40 files changed, 1455 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/377fb897/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --cc streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index 8d66847,1bb565d..33c5827
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@@ -119,15 -117,11 +119,14 @@@ public class StreamsProcessorTask exten
Thread.currentThread().interrupt();
}
if(datum != null) {
+ this.counter.incrementReceivedCount();
try {
+ long startTime = System.currentTimeMillis();
List<StreamsDatum> output = this.processor.process(datum);
+ this.counter.addTime(System.currentTimeMillis() - startTime);
if(output != null) {
for(StreamsDatum outDatum : output) {
- super.addToOutgoingQueue(datum);
- this.counter.incrementEmittedCount();
+ super.addToOutgoingQueue(outDatum);
statusCounter.incrementStatus(DatumStatus.SUCCESS);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/377fb897/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/377fb897/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
----------------------------------------------------------------------
diff --cc streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
index f62250d,2d28602..a0e28cd
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
@@@ -42,18 -38,12 +43,13 @@@ import static org.junit.Assert.*
public class BasicTasksTest {
+ private static final String MBEAN_ID = "test_bean";
-
- /**
- * Remove registered mbeans from previous tests
- * @throws Exception
- */
@After
- public void unregisterMXBean() throws Exception {
+ public void removeLocalMBeans() {
try {
- ManagementFactory.getPlatformMBeanServer().unregisterMBean(new ObjectName(String.format(StreamsTaskCounter.NAME_TEMPLATE, MBEAN_ID)));
- } catch (InstanceNotFoundException ife) {
- //No-op
+ ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
+ } catch (Exception e) {
+ //No op. proceed to next test
}
}
[08/32] incubator-streams git commit: a few package name changes
working twitterurlapiprocess which uses streams-http (w/o authentication)
working people pattern processors which use streams-http (w/ authentication)
Posted by sb...@apache.org.
a few package name changes
working twitterurlapiprocess which uses streams-http (w/o authentication)
working people pattern processors which use streams-http (w/ authentication)
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/0c8e67ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/0c8e67ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/0c8e67ce
Branch: refs/heads/STREAMS-212
Commit: 0c8e67ce26e131282aaaa03815ad6daefd684346
Parents: ad5f90c
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Oct 12 23:13:29 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Oct 12 23:13:29 2014 -0500
----------------------------------------------------------------------
.../http/processor/SimpleHTTPGetProcessor.java | 50 +++++++++++++++-----
.../peoplepattern/AccountTypeProcessor.java | 7 ++-
.../peoplepattern/DemographicsProcessor.java | 7 ++-
.../streams-provider-twitter/pom.xml | 2 +-
.../processor/TwitterUrlApiProcessor.java | 21 ++++----
.../provider/TwitterTimelineProvider.java | 8 ++--
.../apache/streams/data/util/ExtensionUtil.java | 37 ++++++---------
7 files changed, 77 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
index 0d17cc6..d3d4429 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
@@ -21,6 +21,7 @@ import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.data.util.ExtensionUtil;
import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.ActivityObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -99,15 +100,30 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
}
}
- /**
- Override this to place result in non-standard location on document
- */
- protected ObjectNode getEntityToExtend(ObjectNode rootDocument) {
+
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ActivityObject getEntityToExtend(ObjectNode rootDocument) {
+
+ if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
+ return mapper.convertValue(rootDocument, ActivityObject.class);
+ else
+ return mapper.convertValue(rootDocument.get(this.configuration.getEntity().toString()), ActivityObject.class);
+
+ }
+
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode setEntityToExtend(ObjectNode rootDocument, ActivityObject activityObject) {
if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
- return rootDocument;
+ return mapper.convertValue(activityObject, ObjectNode.class);
else
- return (ObjectNode) rootDocument.get(this.configuration.getEntity().toString());
+ rootDocument.set(this.configuration.getEntity().toString(), mapper.convertValue(activityObject, ObjectNode.class));
+
+ return rootDocument;
}
@@ -150,9 +166,6 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
try {
response.close();
} catch (IOException e) {}
- try {
- httpclient.close();
- } catch (IOException e) {}
}
if( entityString == null )
@@ -162,12 +175,12 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
ObjectNode extensionFragment = prepareExtensionFragment(entityString);
- ObjectNode extensionEntity = getEntityToExtend(rootDocument);
-
- ExtensionUtil.ensureExtensions(extensionEntity);
+ ActivityObject extensionEntity = getEntityToExtend(rootDocument);
ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
+ rootDocument = setEntityToExtend(rootDocument, extensionEntity);
+
entry.setDocument(rootDocument);
result.add(entry);
@@ -220,5 +233,18 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
@Override
public void cleanUp() {
LOGGER.info("shutting down SimpleHTTPGetProcessor");
+ try {
+ httpclient.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ httpclient.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ httpclient = null;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
index d180b7f..edcf4d3 100644
--- a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
@@ -18,7 +18,6 @@
package org.apache.streams.peoplepattern;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Maps;
import org.apache.streams.components.http.HttpConfigurator;
import org.apache.streams.components.http.HttpProcessorConfiguration;
@@ -27,6 +26,7 @@ import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.data.util.ExtensionUtil;
import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
import org.apache.streams.pojo.json.Actor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,10 +62,9 @@ public class AccountTypeProcessor extends SimpleHTTPGetProcessor {
@Override
protected Map<String, String> prepareParams(StreamsDatum entry) {
Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
- //Actor actor = mapper.convertValue(entry.getDocument(), Actor.class);
Actor actor = activity.getActor();
- ObjectNode actorObjectNode = mapper.convertValue(actor, ObjectNode.class);
- String username = (String) ExtensionUtil.getExtension(actorObjectNode, "screenName");
+ ActivityObject actorObject = mapper.convertValue(actor, ActivityObject.class);
+ String username = (String) ExtensionUtil.getExtension(actorObject, "screenName");
Map<String, String> params = Maps.newHashMap();
params.put("id", actor.getId());
params.put("name", actor.getDisplayName());
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
index 6ffbb9b..60db379 100644
--- a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
@@ -18,7 +18,6 @@
package org.apache.streams.peoplepattern;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Maps;
import org.apache.streams.components.http.HttpConfigurator;
import org.apache.streams.components.http.HttpProcessorConfiguration;
@@ -27,6 +26,7 @@ import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.data.util.ExtensionUtil;
import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
import org.apache.streams.pojo.json.Actor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,10 +62,9 @@ public class DemographicsProcessor extends SimpleHTTPGetProcessor {
@Override
protected Map<String, String> prepareParams(StreamsDatum entry) {
Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
- //Actor actor = mapper.convertValue(entry.getDocument(), Actor.class);
Actor actor = activity.getActor();
- ObjectNode actorObjectNode = mapper.convertValue(actor, ObjectNode.class);
- String username = (String) ExtensionUtil.getExtension(actorObjectNode, "screenName");
+ ActivityObject actorObject = mapper.convertValue(actor, ActivityObject.class);
+ String username = (String) ExtensionUtil.getExtension(actorObject, "screenName");
Map<String, String> params = Maps.newHashMap();
params.put("id", actor.getId());
params.put("name", actor.getDisplayName());
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
index f0d65f8..604e5a7 100644
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@ -55,7 +55,7 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-processor-http</artifactId>
+ <artifactId>streams-http</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
index 438937f..54e1369 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
@@ -1,11 +1,10 @@
package org.apache.streams.twitter.processor;
import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.streams.components.http.HttpProcessorConfiguration;
-import org.apache.streams.components.http.SimpleHTTPGetProcessor;
-import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.components.http.processor.SimpleHTTPGetProcessor;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.pojo.json.Activity;
@@ -21,23 +20,27 @@ public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements St
public TwitterUrlApiProcessor() {
super();
this.configuration.setHostname("urls.api.twitter.com");
- this.configuration.setResourceUri("/1/urls/count.json");
+ this.configuration.setResourcePath("/1/urls/count.json");
+ this.configuration.setEntity(HttpProcessorConfiguration.Entity.ACTIVITY);
this.configuration.setExtension("twitter_url_count");
}
public TwitterUrlApiProcessor(HttpProcessorConfiguration processorConfiguration) {
super(processorConfiguration);
this.configuration.setHostname("urls.api.twitter.com");
- this.configuration.setResourceUri("/1/urls/count.json");
+ this.configuration.setResourcePath("/1/urls/count.json");
+ this.configuration.setEntity(HttpProcessorConfiguration.Entity.ACTIVITY);
this.configuration.setExtension("twitter_url_count");
}
@Override
public List<StreamsDatum> process(StreamsDatum entry) {
Preconditions.checkArgument(entry.getDocument() instanceof Activity);
- Activity activity = mapper.convertValue(entry, Activity.class);
- Preconditions.checkArgument(!Strings.isNullOrEmpty(activity.getUrl()));
- return super.process(entry);
+ Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
+ if( activity.getLinks() != null && activity.getLinks().size() > 0)
+ return super.process(entry);
+ else
+ return Lists.newArrayList(entry);
}
@Override
@@ -45,7 +48,7 @@ public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements St
Map<String, String> params = Maps.newHashMap();
- params.put("url", mapper.convertValue(entry, Activity.class).getUrl());
+ params.put("url", mapper.convertValue(entry.getDocument(), Activity.class).getLinks().get(0));
return params;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index ae755c2..86395a2 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -19,9 +19,9 @@
package org.apache.streams.twitter.provider;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Queues;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.streams.core.DatumStatusCounter;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProvider;
@@ -31,14 +31,16 @@ import org.apache.streams.util.ComponentUtils;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
import twitter4j.*;
import twitter4j.conf.ConfigurationBuilder;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.*;
-import java.util.concurrent.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0c8e67ce/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
----------------------------------------------------------------------
diff --git a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
index a8d068a..7ce013c 100644
--- a/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
+++ b/streams-pojo-extensions/src/main/java/org/apache/streams/data/util/ExtensionUtil.java
@@ -1,7 +1,6 @@
package org.apache.streams.data.util;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Maps;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.ActivityObject;
@@ -41,38 +40,33 @@ public class ExtensionUtil {
private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- public static Map<String, Object> getExtensions(ObjectNode object) {
+ public static Map<String, Object> getExtensions(ActivityObject object) {
ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ Map<String,Object> extensions = ensureExtensions(object);
return extensions;
}
- public static Object getExtension(ObjectNode object, String key) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ public static Object getExtension(ActivityObject object, String key) {
+ Map<String,Object> extensions = ensureExtensions(object);
return extensions.get(key);
}
- public static void setExtensions(ObjectNode object, Map<String, Object> extensions) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- activityObject.setAdditionalProperty(EXTENSION_PROPERTY, extensions);
+ public static void setExtensions(ActivityObject object, Map<String, Object> extensions) {
+ object.setAdditionalProperty(EXTENSION_PROPERTY, extensions);
};
- public static void addExtension(ObjectNode object, String key, Object extension) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ public static void addExtension(ActivityObject object, String key, Object extension) {
+ Map<String,Object> extensions = ensureExtensions(object);
extensions.put(key, extension);
};
- public static void addExtensions(ObjectNode object, Map<String, Object> extensions) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
+ public static void addExtensions(ActivityObject object, Map<String, Object> extensions) {
for( Map.Entry<String, Object> item : extensions.entrySet())
- activityObject.getAdditionalProperties().put(item.getKey(), item.getValue());
+ addExtension(object, item.getKey(), item.getValue());
};
- public static void removeExtension(ObjectNode object, String key) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ public static void removeExtension(ActivityObject object, String key) {
+ Map<String,Object> extensions = ensureExtensions(object);
extensions.remove(key);
};
@@ -82,13 +76,12 @@ public class ExtensionUtil {
* @return the Map representing the extensions property
*/
@SuppressWarnings("unchecked")
- public static Map<String, Object> ensureExtensions(ObjectNode object) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- Map<String,Object> extensions = (Map<String,Object>) activityObject.getAdditionalProperties().get(EXTENSION_PROPERTY);
+ public static Map<String, Object> ensureExtensions(ActivityObject object) {
+ Map<String,Object> extensions = (Map<String,Object>) object.getAdditionalProperties().get(EXTENSION_PROPERTY);
if(extensions == null) {
extensions = Maps.newHashMap();
setExtensions(object, extensions);
}
- return getExtensions(object);
+ return extensions;
}
}
[25/32] incubator-streams git commit: Merge branch 'STREAMS-168'
Posted by sb...@apache.org.
Merge branch 'STREAMS-168'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ab9c6963
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ab9c6963
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ab9c6963
Branch: refs/heads/STREAMS-212
Commit: ab9c6963d8d3524ab88703a73a6c0e05842a5fee
Parents: e83659c d2ffe62
Author: sblackmon <sb...@apache.org>
Authored: Fri Nov 7 10:48:50 2014 -0800
Committer: sblackmon <sb...@apache.org>
Committed: Fri Nov 7 10:48:50 2014 -0800
----------------------------------------------------------------------
pom.xml | 2 +
streams-components/pom.xml | 62 +++++
streams-components/streams-http/README.md | 16 ++
streams-components/streams-http/pom.xml | 153 +++++++++++
.../components/http/HttpConfigurator.java | 62 +++++
.../http/processor/SimpleHTTPGetProcessor.java | 268 +++++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 230 ++++++++++++++++
.../components/http/HttpConfiguration.json | 50 ++++
.../http/HttpProcessorConfiguration.json | 28 ++
.../http/HttpProviderConfiguration.json | 18 ++
streams-contrib/pom.xml | 5 +-
.../streams-processor-peoplepattern/pom.xml | 138 ++++++++++
.../peoplepattern/AccountTypeProcessor.java | 75 ++++++
.../peoplepattern/DemographicsProcessor.java | 76 ++++++
.../streams/peoplepattern/AccountType.json | 27 ++
.../streams/peoplepattern/Demographics.json | 60 +++++
.../resources/templates/peoplepatternactor.json | 25 ++
.../api/FacebookPostActivitySerializer.java | 1 -
.../streams-provider-twitter/pom.xml | 5 +
.../processor/TwitterUrlApiProcessor.java | 73 +++++
.../provider/TwitterTimelineProvider.java | 7 +-
streams-pojo-extensions/pom.xml | 64 +++++
.../apache/streams/data/util/ExtensionUtil.java | 108 ++++++++
.../apache/streams/data/util/ActivityUtil.java | 14 +-
.../org/apache/streams/pojo/json/activity.json | 3 +-
.../org/apache/streams/pojo/json/object.json | 2 +-
26 files changed, 1559 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ab9c6963/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ab9c6963/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ab9c6963/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ab9c6963/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
[32/32] incubator-streams git commit: revamped testing,
more cases more conditions per test
Posted by sb...@apache.org.
revamped testing, more cases more conditions per test
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/99e70b48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/99e70b48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/99e70b48
Branch: refs/heads/STREAMS-212
Commit: 99e70b48c426a2753f6e385c561df7286f15e3f6
Parents: dabc511
Author: sblackmon <sb...@apache.org>
Authored: Tue Nov 11 13:50:18 2014 -0600
Committer: sblackmon <sb...@apache.org>
Committed: Tue Nov 11 13:50:18 2014 -0600
----------------------------------------------------------------------
.../serializer/DatasiftEventClassifier.java | 2 +-
.../DatasiftTweetActivitySerializer.java | 272 -------------------
.../DatasiftTwitterActivitySerializer.java | 272 +++++++++++++++++++
.../DatasiftActivitySerializerTest.java | 80 ++----
.../serializer/DatasiftEventClassifierTest.java | 14 +-
...DatasiftInstagramActivitySerializerTest.java | 43 +++
...tasiftInteractionActivitySerializerTest.java | 48 ++++
.../DatasiftTwitterActivitySerializerTest.java | 43 +++
8 files changed, 437 insertions(+), 337 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/99e70b48/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
index 0169f17..7d7d547 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
@@ -43,7 +43,7 @@ public class DatasiftEventClassifier {
public static ActivitySerializer bestSerializer(Datasift event) {
if(event.getTwitter() != null) {
- return DatasiftTweetActivitySerializer.getInstance();
+ return DatasiftTwitterActivitySerializer.getInstance();
} else if(event.getInstagram() != null) {
return DatasiftInstagramActivitySerializer.getInstance();
} else {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/99e70b48/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
deleted file mode 100644
index 6fd19e7..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTweetActivitySerializer.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-package org.apache.streams.datasift.serializer;
-
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.interaction.Author;
-import org.apache.streams.datasift.interaction.Interaction;
-import org.apache.streams.datasift.twitter.DatasiftTwitterUser;
-import org.apache.streams.datasift.twitter.Retweet;
-import org.apache.streams.datasift.twitter.Twitter;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.twitter.serializer.util.TwitterActivityUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
-
-/**
- *
- */
-public class DatasiftTweetActivitySerializer extends DatasiftInteractionActivitySerializer {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftTweetActivitySerializer.class);
-
- private static DatasiftTweetActivitySerializer instance = new DatasiftTweetActivitySerializer();
-
- public static DatasiftTweetActivitySerializer getInstance() {
- return instance;
- }
-
- @Override
- public Activity convert(Datasift event) {
- Activity activity = new Activity();
- Twitter twitter = event.getTwitter();
- boolean retweet = twitter.getRetweet() != null;
-
- activity.setActor(buildActor(event, twitter)); //TODO
- if(retweet) {
- activity.setVerb("share");
- } else {
- activity.setVerb("post");
- }
- activity.setObject(buildActivityObject(event.getInteraction()));
- activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
- activity.setTarget(buildTarget(event.getInteraction()));
- activity.setPublished(event.getInteraction().getCreatedAt());
- activity.setGenerator(buildGenerator(event.getInteraction()));
- activity.setIcon(getIcon(event.getInteraction()));
- activity.setProvider(TwitterActivityUtil.getProvider());
- activity.setTitle(event.getInteraction().getTitle());
- activity.setContent(event.getInteraction().getContent());
- activity.setUrl(event.getInteraction().getLink());
- if(retweet)
- activity.setLinks(getLinks(twitter.getRetweet()));
- else
- activity.setLinks(getLinks(twitter));
- addDatasiftExtension(activity, event);
- if( twitter.getGeo() != null) {
- addLocationExtension(activity, twitter);
- }
- addTwitterExtensions(activity, twitter, event.getInteraction());
- return activity;
- }
-
- /**
- * Get the links from this tweet as a list
- * @param twitter
- * @return the links from the tweet
- */
- public List<String> getLinks(Twitter twitter) {
- return getLinks(twitter.getLinks());
- }
-
- /**
- * Get the links from this tweet as a list
- * @param retweet
- * @return the links from the tweet
- */
- public List<String> getLinks(Retweet retweet) {
- return getLinks(retweet.getLinks());
- }
-
- /**
- * Converts the list of objects to a list of strings
- * @param links
- * @return
- */
- private List<String> getLinks(List<Object> links) {
- if(links == null)
- return Lists.newArrayList();
- List<String> result = Lists.newLinkedList();
- for(Object obj : links) {
- if(obj instanceof String) {
- result.add((String) obj);
- } else {
- LOGGER.warn("Links is not instance of String : {}", obj.getClass().getName());
- }
- }
- return result;
- }
-
- public Actor buildActor(Datasift event, Twitter twitter) {
- DatasiftTwitterUser user = twitter.getUser();
- Actor actor = super.buildActor(event.getInteraction());
- if(user == null) {
- user = twitter.getRetweet().getUser();
- }
-
- actor.setDisplayName(user.getName());
- actor.setId(formatId(Optional.fromNullable(
- user.getIdStr())
- .or(Optional.of(user.getId().toString()))
- .orNull()));
- actor.setSummary(user.getDescription());
- try {
- actor.setPublished(user.getCreatedAt());
- } catch (Exception e) {
- LOGGER.warn("Exception trying to parse date : {}", e);
- }
-
- if(user.getUrl() != null) {
- actor.setUrl(user.getUrl());
- }
-
- Map<String, Object> extensions = new HashMap<String,Object>();
- extensions.put("location", user.getLocation());
- extensions.put("posts", user.getStatusesCount());
- extensions.put("followers", user.getFollowersCount());
- extensions.put("screenName", user.getScreenName());
- if(user.getAdditionalProperties() != null) {
- extensions.put("favorites", user.getFavouritesCount());
- }
-
- Image profileImage = new Image();
- String profileUrl = null;
- Author author = event.getInteraction().getAuthor();
- if( author != null )
- profileUrl = author.getAvatar();
- if(profileUrl == null && user.getProfileImageUrlHttps() != null) {
- Object url = user.getProfileImageUrlHttps();
- if(url instanceof String)
- profileUrl = (String) url;
- }
- if(profileUrl == null) {
- profileUrl = user.getProfileImageUrl();
- }
- profileImage.setUrl(profileUrl);
- actor.setImage(profileImage);
-
- actor.setAdditionalProperty("extensions", extensions);
- return actor;
- }
-
- public void addLocationExtension(Activity activity, Twitter twitter) {
- Map<String, Object> extensions = ensureExtensions(activity);
- Map<String, Object> location = Maps.newHashMap();
- double[] coordiantes = new double[] { twitter.getGeo().getLongitude(), twitter.getGeo().getLatitude() };
- Map<String, Object> coords = Maps.newHashMap();
- coords.put("coordinates", coordiantes);
- coords.put("type", "geo_point");
- location.put("coordinates", coords);
- extensions.put("location", location);
- }
-
- public void addTwitterExtensions(Activity activity, Twitter twitter, Interaction interaction) {
- Retweet retweet = twitter.getRetweet();
- Map<String, Object> extensions = ensureExtensions(activity);
- List<String> hashTags = Lists.newLinkedList();
- List<Object> hts = Lists.newLinkedList();
- if(twitter.getHashtags() != null) {
- hts = twitter.getHashtags();
- } else if (retweet != null) {
- hts = retweet.getHashtags();
- }
- if(hts != null) {
- for(Object ht : twitter.getHashtags()) {
- if(ht instanceof String) {
- hashTags.add((String) ht);
- } else {
- LOGGER.warn("Hashtag was not instance of String : {}", ht.getClass().getName());
- }
- }
- }
- extensions.put("hashtags", hashTags);
-
-
- if(retweet != null) {
- Map<String, Object> rebroadcasts = Maps.newHashMap();
- rebroadcasts.put("perspectival", true);
- rebroadcasts.put("count", retweet.getCount());
- extensions.put("rebroadcasts", rebroadcasts);
- }
-
- if(interaction.getAdditionalProperties() != null) {
- ArrayList<Map<String,Object>> userMentions = createUserMentions(interaction);
-
- if(userMentions.size() > 0)
- extensions.put("user_mentions", userMentions);
- }
-
- extensions.put("keywords", interaction.getContent());
- }
-
- /**
- * Returns an ArrayList of all UserMentions in this interaction
- * Note: The ID list and the handle lists do not necessarily correspond 1:1 for this provider
- * If those lists are the same size, then they will be merged into individual UserMention
- * objects. However, if they are not the same size, a new UserMention object will be created
- * for each entry in both lists.
- *
- * @param interaction
- * @return
- */
- private ArrayList<Map<String,Object>> createUserMentions(Interaction interaction) {
- ArrayList<String> mentions = (ArrayList<String>) interaction.getAdditionalProperties().get("mentions");
- ArrayList<Long> mentionIds = (ArrayList<Long>) interaction.getAdditionalProperties().get("mention_ids");
- ArrayList<Map<String,Object>> userMentions = new ArrayList<Map<String,Object>>();
-
- if(mentions != null && !mentions.isEmpty()) {
- for(int x = 0; x < mentions.size(); x ++) {
- Map<String, Object> actor = new HashMap<String, Object>();
- actor.put("displayName", mentions.get(x));
- actor.put("handle", mentions.get(x));
-
- userMentions.add(actor);
- }
- }
- if(mentionIds != null && !mentionIds.isEmpty()) {
- for(int x = 0; x < mentionIds.size(); x ++) {
- Map<String, Object> actor = new HashMap<String, Object>();
- actor.put("id", "id:twitter:" + mentionIds.get(x));
-
- userMentions.add(actor);
- }
- }
-
- return userMentions;
- }
-
- public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/99e70b48/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java
new file mode 100644
index 0000000..8ac84f6
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java
@@ -0,0 +1,272 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+package org.apache.streams.datasift.serializer;
+
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.streams.datasift.Datasift;
+import org.apache.streams.datasift.interaction.Author;
+import org.apache.streams.datasift.interaction.Interaction;
+import org.apache.streams.datasift.twitter.DatasiftTwitterUser;
+import org.apache.streams.datasift.twitter.Retweet;
+import org.apache.streams.datasift.twitter.Twitter;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.apache.streams.pojo.json.Image;
+import org.apache.streams.twitter.serializer.util.TwitterActivityUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
+
+/**
+ *
+ */
+public class DatasiftTwitterActivitySerializer extends DatasiftInteractionActivitySerializer {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftTwitterActivitySerializer.class);
+
+ private static DatasiftTwitterActivitySerializer instance = new DatasiftTwitterActivitySerializer();
+
+ public static DatasiftTwitterActivitySerializer getInstance() {
+ return instance;
+ }
+
+ @Override
+ public Activity convert(Datasift event) {
+ Activity activity = new Activity();
+ Twitter twitter = event.getTwitter();
+ boolean retweet = twitter.getRetweet() != null;
+
+ activity.setActor(buildActor(event, twitter)); //TODO
+ if(retweet) {
+ activity.setVerb("share");
+ } else {
+ activity.setVerb("post");
+ }
+ activity.setObject(buildActivityObject(event.getInteraction()));
+ activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
+ activity.setTarget(buildTarget(event.getInteraction()));
+ activity.setPublished(event.getInteraction().getCreatedAt());
+ activity.setGenerator(buildGenerator(event.getInteraction()));
+ activity.setIcon(getIcon(event.getInteraction()));
+ activity.setProvider(TwitterActivityUtil.getProvider());
+ activity.setTitle(event.getInteraction().getTitle());
+ activity.setContent(event.getInteraction().getContent());
+ activity.setUrl(event.getInteraction().getLink());
+ if(retweet)
+ activity.setLinks(getLinks(twitter.getRetweet()));
+ else
+ activity.setLinks(getLinks(twitter));
+ addDatasiftExtension(activity, event);
+ if( twitter.getGeo() != null) {
+ addLocationExtension(activity, twitter);
+ }
+ addTwitterExtensions(activity, twitter, event.getInteraction());
+ return activity;
+ }
+
+ /**
+ * Get the links from this tweet as a list
+ * @param twitter
+ * @return the links from the tweet
+ */
+ public List<String> getLinks(Twitter twitter) {
+ return getLinks(twitter.getLinks());
+ }
+
+ /**
+ * Get the links from this tweet as a list
+ * @param retweet
+ * @return the links from the tweet
+ */
+ public List<String> getLinks(Retweet retweet) {
+ return getLinks(retweet.getLinks());
+ }
+
+ /**
+ * Converts the list of objects to a list of strings
+ * @param links
+ * @return
+ */
+ private List<String> getLinks(List<Object> links) {
+ if(links == null)
+ return Lists.newArrayList();
+ List<String> result = Lists.newLinkedList();
+ for(Object obj : links) {
+ if(obj instanceof String) {
+ result.add((String) obj);
+ } else {
+ LOGGER.warn("Links is not instance of String : {}", obj.getClass().getName());
+ }
+ }
+ return result;
+ }
+
+ public Actor buildActor(Datasift event, Twitter twitter) {
+ DatasiftTwitterUser user = twitter.getUser();
+ Actor actor = super.buildActor(event.getInteraction());
+ if(user == null) {
+ user = twitter.getRetweet().getUser();
+ }
+
+ actor.setDisplayName(user.getName());
+ actor.setId(formatId(Optional.fromNullable(
+ user.getIdStr())
+ .or(Optional.of(user.getId().toString()))
+ .orNull()));
+ actor.setSummary(user.getDescription());
+ try {
+ actor.setPublished(user.getCreatedAt());
+ } catch (Exception e) {
+ LOGGER.warn("Exception trying to parse date : {}", e);
+ }
+
+ if(user.getUrl() != null) {
+ actor.setUrl(user.getUrl());
+ }
+
+ Map<String, Object> extensions = new HashMap<String,Object>();
+ extensions.put("location", user.getLocation());
+ extensions.put("posts", user.getStatusesCount());
+ extensions.put("followers", user.getFollowersCount());
+ extensions.put("screenName", user.getScreenName());
+ if(user.getAdditionalProperties() != null) {
+ extensions.put("favorites", user.getFavouritesCount());
+ }
+
+ Image profileImage = new Image();
+ String profileUrl = null;
+ Author author = event.getInteraction().getAuthor();
+ if( author != null )
+ profileUrl = author.getAvatar();
+ if(profileUrl == null && user.getProfileImageUrlHttps() != null) {
+ Object url = user.getProfileImageUrlHttps();
+ if(url instanceof String)
+ profileUrl = (String) url;
+ }
+ if(profileUrl == null) {
+ profileUrl = user.getProfileImageUrl();
+ }
+ profileImage.setUrl(profileUrl);
+ actor.setImage(profileImage);
+
+ actor.setAdditionalProperty("extensions", extensions);
+ return actor;
+ }
+
+ public void addLocationExtension(Activity activity, Twitter twitter) {
+ Map<String, Object> extensions = ensureExtensions(activity);
+ Map<String, Object> location = Maps.newHashMap();
+ double[] coordiantes = new double[] { twitter.getGeo().getLongitude(), twitter.getGeo().getLatitude() };
+ Map<String, Object> coords = Maps.newHashMap();
+ coords.put("coordinates", coordiantes);
+ coords.put("type", "geo_point");
+ location.put("coordinates", coords);
+ extensions.put("location", location);
+ }
+
+ public void addTwitterExtensions(Activity activity, Twitter twitter, Interaction interaction) {
+ Retweet retweet = twitter.getRetweet();
+ Map<String, Object> extensions = ensureExtensions(activity);
+ List<String> hashTags = Lists.newLinkedList();
+ List<Object> hts = Lists.newLinkedList();
+ if(twitter.getHashtags() != null) {
+ hts = twitter.getHashtags();
+ } else if (retweet != null) {
+ hts = retweet.getHashtags();
+ }
+ if(hts != null) {
+ for(Object ht : twitter.getHashtags()) {
+ if(ht instanceof String) {
+ hashTags.add((String) ht);
+ } else {
+ LOGGER.warn("Hashtag was not instance of String : {}", ht.getClass().getName());
+ }
+ }
+ }
+ extensions.put("hashtags", hashTags);
+
+
+ if(retweet != null) {
+ Map<String, Object> rebroadcasts = Maps.newHashMap();
+ rebroadcasts.put("perspectival", true);
+ rebroadcasts.put("count", retweet.getCount());
+ extensions.put("rebroadcasts", rebroadcasts);
+ }
+
+ if(interaction.getAdditionalProperties() != null) {
+ ArrayList<Map<String,Object>> userMentions = createUserMentions(interaction);
+
+ if(userMentions.size() > 0)
+ extensions.put("user_mentions", userMentions);
+ }
+
+ extensions.put("keywords", interaction.getContent());
+ }
+
+ /**
+ * Returns an ArrayList of all UserMentions in this interaction
+ * Note: The ID list and the handle lists do not necessarily correspond 1:1 for this provider
+ * If those lists are the same size, then they will be merged into individual UserMention
+ * objects. However, if they are not the same size, a new UserMention object will be created
+ * for each entry in both lists.
+ *
+ * @param interaction
+ * @return
+ */
+ private ArrayList<Map<String,Object>> createUserMentions(Interaction interaction) {
+ ArrayList<String> mentions = (ArrayList<String>) interaction.getAdditionalProperties().get("mentions");
+ ArrayList<Long> mentionIds = (ArrayList<Long>) interaction.getAdditionalProperties().get("mention_ids");
+ ArrayList<Map<String,Object>> userMentions = new ArrayList<Map<String,Object>>();
+
+ if(mentions != null && !mentions.isEmpty()) {
+ for(int x = 0; x < mentions.size(); x ++) {
+ Map<String, Object> actor = new HashMap<String, Object>();
+ actor.put("displayName", mentions.get(x));
+ actor.put("handle", mentions.get(x));
+
+ userMentions.add(actor);
+ }
+ }
+ if(mentionIds != null && !mentionIds.isEmpty()) {
+ for(int x = 0; x < mentionIds.size(); x ++) {
+ Map<String, Object> actor = new HashMap<String, Object>();
+ actor.put("id", "id:twitter:" + mentionIds.get(x));
+
+ userMentions.add(actor);
+ }
+ }
+
+ return userMentions;
+ }
+
+ public static String formatId(String... idparts) {
+ return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/99e70b48/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
index 162526b..8f7ad43 100644
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java
@@ -4,10 +4,13 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
+import org.apache.streams.data.ActivitySerializer;
+import org.apache.streams.datasift.Datasift;
import org.apache.streams.datasift.util.StreamsDatasiftMapper;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.pojo.json.Activity;
import org.apache.streams.pojo.json.Actor;
+import org.junit.Before;
import org.junit.Test;
import java.util.Scanner;
@@ -17,18 +20,27 @@ import static org.junit.Assert.assertNotNull;
public class DatasiftActivitySerializerTest {
- private static final DatasiftActivitySerializer SERIALIZER = new DatasiftActivitySerializer();
+ protected ActivitySerializer SERIALIZER;
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(Lists.newArrayList(StreamsDatasiftMapper.DATASIFT_FORMAT));
+ protected static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(Lists.newArrayList(StreamsDatasiftMapper.DATASIFT_FORMAT));
+
+ @Before
+ public void initSerializer() {
+ SERIALIZER = new DatasiftActivitySerializer();
+ }
@Test
- public void testGeneralConversion() throws Exception {
+ public void testConversion() throws Exception {
Scanner scanner = new Scanner(DatasiftActivitySerializerTest.class.getResourceAsStream("/rand_sample_datasift_json.txt"));
String line = null;
while(scanner.hasNextLine()) {
try {
line = scanner.nextLine();
- testGeneralConversion(line);
+ Datasift item = MAPPER.readValue(line, Datasift.class);
+ testConversion(item);
+ String json = MAPPER.writeValueAsString(item);
+ testDeserNoNull(json);
+ testDeserNoAddProps(json);
} catch (Exception e) {
System.err.println(line);
throw e;
@@ -36,59 +48,27 @@ public class DatasiftActivitySerializerTest {
}
}
- @Test
- public void testTwitterConversion() throws Exception {
- Scanner scanner = new Scanner(DatasiftActivitySerializerTest.class.getResourceAsStream("/twitter_datasift_json.txt"));
- String line = null;
- while(scanner.hasNextLine()) {
- line = scanner.nextLine();
- testGeneralConversion(line);
- testDeserNoNull(line);
- testDeserNoAddProps(line);
-
- System.out.println("ORIGINAL -> "+line);
- System.out.println("ACTIVITY -> "+MAPPER.writeValueAsString(SERIALIZER.deserialize(line)));
- System.out.println("NODE -> "+MAPPER.convertValue(SERIALIZER.deserialize(line), JsonNode.class));
- }
- }
-
- @Test
- public void testInstagramConversion() throws Exception {
- Scanner scanner = new Scanner(DatasiftActivitySerializerTest.class.getResourceAsStream("/instagram_datasift_json.txt"));
- String line = null;
- while(scanner.hasNextLine()) {
- line = scanner.nextLine();
- testGeneralConversion(line);
- System.out.println("ORIGINAL -> "+line);
- System.out.println("ACTIVITY -> "+MAPPER.writeValueAsString(SERIALIZER.deserialize(line)));
- System.out.println("NODE -> "+MAPPER.convertValue(SERIALIZER.deserialize(line), JsonNode.class));
- }
- }
-
/**
* Test that the minimum number of things that an activity has
- * @param json
+ * @param item
*/
- private void testGeneralConversion(String json) throws Exception {
- Activity activity = SERIALIZER.deserialize(json);
- assertNotNull(json, activity.getId());
- assertNotNull(json, activity.getPublished());
- assertNotNull(json, activity.getProvider());
- assertNotNull(json, activity.getUrl());
- assertNotNull(json, activity.getVerb());
+ protected void testConversion(Datasift item) throws Exception {
+ Activity activity = SERIALIZER.deserialize(item);
+ assertNotNull("activity.id", activity.getId());
+ assertNotNull("activity.published", activity.getPublished());
+ assertNotNull("activity.provider", activity.getProvider());
+ assertNotNull("activity.url", activity.getUrl());
+ assertNotNull("activity.verb", activity.getVerb());
Actor actor = activity.getActor();
- assertNotNull(json, actor);
-
+ assertNotNull("activity.actor", actor);
}
/**
* Test that null fields are not present
* @param json
*/
- private void testDeserNoNull(String json) throws Exception {
- Activity ser = SERIALIZER.deserialize(json);
- String deser = MAPPER.writeValueAsString(ser);
- int nulls = StringUtils.countMatches(deser, ":null");
+ protected void testDeserNoNull(String json) throws Exception {
+ int nulls = StringUtils.countMatches(json, ":null");
assertEquals(0l, (long)nulls);
}
@@ -97,10 +77,8 @@ public class DatasiftActivitySerializerTest {
* Test that null fields are not present
* @param json
*/
- private void testDeserNoAddProps(String json) throws Exception {
- Activity ser = SERIALIZER.deserialize(json);
- String deser = MAPPER.writeValueAsString(ser);
- int nulls = StringUtils.countMatches(deser, "additionalProperties:{");
+ protected void testDeserNoAddProps(String json) throws Exception {
+ int nulls = StringUtils.countMatches(json, "additionalProperties:{");
assertEquals(0l, (long)nulls);
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/99e70b48/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java
index 2004654..fda57c4 100644
--- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java
@@ -18,25 +18,13 @@
package org.apache.streams.datasift.serializer;
-import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
-import org.apache.streams.data.ActivitySerializer;
import org.apache.streams.datasift.Datasift;
import org.apache.streams.datasift.instagram.Instagram;
import org.apache.streams.datasift.twitter.Twitter;
import org.apache.streams.datasift.util.StreamsDatasiftMapper;
import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.twitter.provider.TwitterEventClassifier;
-import org.apache.streams.twitter.serializer.TwitterJsonDeleteActivitySerializer;
-import org.apache.streams.twitter.serializer.TwitterJsonRetweetActivitySerializer;
-import org.apache.streams.twitter.serializer.TwitterJsonTweetActivitySerializer;
-import org.apache.streams.twitter.serializer.TwitterJsonUserActivitySerializer;
-import org.junit.Assert;
import org.junit.Test;
import java.util.Scanner;
@@ -56,7 +44,7 @@ public class DatasiftEventClassifierTest {
line = scanner.nextLine();
Datasift datasift = MAPPER.readValue(line, Datasift.class);
assert(DatasiftEventClassifier.detectClass(datasift) == Twitter.class);
- assert(DatasiftEventClassifier.bestSerializer(datasift) instanceof DatasiftTweetActivitySerializer);
+ assert(DatasiftEventClassifier.bestSerializer(datasift) instanceof DatasiftTwitterActivitySerializer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/99e70b48/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerTest.java
new file mode 100644
index 0000000..5350d74
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerTest.java
@@ -0,0 +1,43 @@
+package org.apache.streams.datasift.serializer;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.datasift.Datasift;
+import org.apache.streams.datasift.util.StreamsDatasiftMapper;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Scanner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class DatasiftInstagramActivitySerializerTest extends DatasiftActivitySerializerTest {
+
+ @Before
+ @Override
+ public void initSerializer() {
+ SERIALIZER = new DatasiftInstagramActivitySerializer();
+ }
+
+ @Test
+ @Override
+ public void testConversion() throws Exception {
+ Scanner scanner = new Scanner(DatasiftActivitySerializerTest.class.getResourceAsStream("/instagram_datasift_json.txt"));
+ String line = null;
+ while(scanner.hasNextLine()) {
+ line = scanner.nextLine();
+ Datasift item = MAPPER.readValue(line, Datasift.class);
+ testConversion(item);
+ String json = MAPPER.writeValueAsString(item);
+ testDeserNoNull(json);
+ testDeserNoAddProps(json);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/99e70b48/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerTest.java
new file mode 100644
index 0000000..21d4ebb
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerTest.java
@@ -0,0 +1,48 @@
+package org.apache.streams.datasift.serializer;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.datasift.Datasift;
+import org.apache.streams.datasift.util.StreamsDatasiftMapper;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Scanner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class DatasiftInteractionActivitySerializerTest extends DatasiftActivitySerializerTest {
+
+ @Before
+ @Override
+ public void initSerializer() {
+ SERIALIZER = new DatasiftInteractionActivitySerializer();
+ }
+
+ @Test
+ @Override
+ public void testConversion() throws Exception {
+ Scanner scanner = new Scanner(DatasiftInteractionActivitySerializerTest.class.getResourceAsStream("/rand_sample_datasift_json.txt"));
+ String line = null;
+ while(scanner.hasNextLine()) {
+ try {
+ line = scanner.nextLine();
+ Datasift item = MAPPER.readValue(line, Datasift.class);
+ testConversion(item);
+ String json = MAPPER.writeValueAsString(item);
+ testDeserNoNull(json);
+ testDeserNoAddProps(json);
+ } catch (Exception e) {
+ System.err.println(line);
+ throw e;
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/99e70b48/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerTest.java
new file mode 100644
index 0000000..33b1f77
--- /dev/null
+++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerTest.java
@@ -0,0 +1,43 @@
+package org.apache.streams.datasift.serializer;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
+import org.apache.streams.datasift.Datasift;
+import org.apache.streams.datasift.util.StreamsDatasiftMapper;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Scanner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class DatasiftTwitterActivitySerializerTest extends DatasiftActivitySerializerTest {
+
+ @Before
+ @Override
+ public void initSerializer() {
+ SERIALIZER = new DatasiftTwitterActivitySerializer();
+ }
+
+ @Test
+ @Override
+ public void testConversion() throws Exception {
+ Scanner scanner = new Scanner(DatasiftTwitterActivitySerializerTest.class.getResourceAsStream("/twitter_datasift_json.txt"));
+ String line = null;
+ while(scanner.hasNextLine()) {
+ line = scanner.nextLine();
+ Datasift item = MAPPER.readValue(line, Datasift.class);
+ testConversion(item);
+ String json = MAPPER.writeValueAsString(item);
+ testDeserNoNull(json);
+ testDeserNoAddProps(json);
+ }
+ }
+
+}
[31/32] incubator-streams git commit: Merge branch 'master' into
STREAMS-212
Posted by sb...@apache.org.
Merge branch 'master' into STREAMS-212
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/dabc5116
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/dabc5116
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/dabc5116
Branch: refs/heads/STREAMS-212
Commit: dabc51162f1260d8cc7bf8b66902264049876b13
Parents: a6f8626 bfa9466
Author: sblackmon <sb...@apache.org>
Authored: Tue Nov 11 12:55:59 2014 -0600
Committer: sblackmon <sb...@apache.org>
Committed: Tue Nov 11 12:55:59 2014 -0600
----------------------------------------------------------------------
pom.xml | 58 ++++
streams-components/pom.xml | 62 +++++
streams-components/streams-http/README.md | 16 ++
streams-components/streams-http/pom.xml | 153 +++++++++++
.../components/http/HttpConfigurator.java | 62 +++++
.../http/processor/SimpleHTTPGetProcessor.java | 268 +++++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 230 ++++++++++++++++
.../components/http/HttpConfiguration.json | 50 ++++
.../http/HttpProcessorConfiguration.json | 28 ++
.../http/HttpProviderConfiguration.json | 18 ++
streams-contrib/pom.xml | 5 +-
streams-contrib/streams-persist-console/pom.xml | 34 +++
.../streams/console/ConsolePersistReader.java | 11 +-
.../streams/console/ConsolePersistWriter.java | 11 +-
.../streams-processor-peoplepattern/pom.xml | 138 ++++++++++
.../peoplepattern/AccountTypeProcessor.java | 75 ++++++
.../peoplepattern/DemographicsProcessor.java | 76 ++++++
.../streams/peoplepattern/AccountType.json | 27 ++
.../streams/peoplepattern/Demographics.json | 60 +++++
.../resources/templates/peoplepatternactor.json | 25 ++
.../api/FacebookPostActivitySerializer.java | 1 -
.../google-gplus/pom.xml | 32 +++
.../processor/GooglePlusCommentProcessor.java | 87 ++++++
.../gplus/provider/AbstractGPlusProvider.java | 234 ++++++++++++++++
.../gplus/provider/GPlusActivitySerializer.java | 4 +-
.../gplus/provider/GPlusDataCollector.java | 50 ++++
.../provider/GPlusHistoryProviderTask.java | 106 --------
.../google/gplus/provider/GPlusProvider.java | 189 -------------
.../provider/GPlusUserActivityCollector.java | 108 ++++++++
.../provider/GPlusUserActivityProvider.java | 18 ++
.../gplus/provider/GPlusUserDataCollector.java | 79 ++++++
.../gplus/provider/GPlusUserDataProvider.java | 18 ++
.../util/GPlusCommentDeserializer.java | 98 +++++++
.../serializer/util/GooglePlusActivityUtil.java | 50 ++++
.../com/google/gplus/GPlusConfiguration.json | 44 ++-
.../gplus/GooglePlusCommentSerDeTest.java | 114 ++++++++
.../provider/TestAbstractGPlusProvider.java | 82 ++++++
.../TestGPlusUserActivityCollector.java | 268 +++++++++++++++++++
.../provider/TestGPlusUserDataCollector.java | 131 +++++++++
.../resources/google_plus_comments_jsons.txt | 3 +
streams-contrib/streams-provider-google/pom.xml | 6 +
.../streams-provider-twitter/pom.xml | 5 +
.../processor/TwitterUrlApiProcessor.java | 73 +++++
.../provider/TwitterTimelineProvider.java | 7 +-
streams-pojo-extensions/pom.xml | 64 +++++
.../apache/streams/data/util/ExtensionUtil.java | 108 ++++++++
.../apache/streams/data/util/ActivityUtil.java | 14 +-
.../jackson/StreamsDateTimeDeserializer.java | 23 +-
.../streams/jackson/StreamsJacksonMapper.java | 20 ++
.../streams/jackson/StreamsJacksonModule.java | 9 +
.../org/apache/streams/pojo/json/activity.json | 3 +-
.../org/apache/streams/pojo/json/object.json | 2 +-
.../local/builders/LocalStreamBuilder.java | 5 +
.../streams/local/tasks/StreamsMergeTask.java | 7 +
.../local/tasks/StreamsPersistWriterTask.java | 18 +-
.../local/tasks/StreamsProcessorTask.java | 20 +-
.../local/tasks/StreamsProviderTask.java | 15 ++
.../apache/streams/local/tasks/StreamsTask.java | 4 +
.../local/builders/LocalStreamBuilderTest.java | 32 ++-
.../streams/local/tasks/BasicTasksTest.java | 26 +-
.../backoff/AbstractBackOffStrategy.java | 15 +-
61 files changed, 3255 insertions(+), 344 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/dabc5116/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --cc streams-contrib/streams-provider-twitter/pom.xml
index 9c99a92,0bf3fe7..3880135
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@@ -50,14 -50,13 +50,19 @@@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
+ <artifactId>streams-processor-jackson</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
<artifactId>streams-util</artifactId>
<version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-http</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
[04/32] incubator-streams git commit: class need not be abstract
rootDocument is what should exit processor
Posted by sb...@apache.org.
class need not be abstract
rootDocument is what should exit processor
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/a5764609
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/a5764609
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/a5764609
Branch: refs/heads/STREAMS-212
Commit: a576460911464cf260e37886ce840f4e57dd5f30
Parents: b8ccf9f
Author: sblackmon <sb...@apache.org>
Authored: Tue Sep 16 14:24:53 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Tue Sep 16 14:24:53 2014 -0500
----------------------------------------------------------------------
.../apache/streams/components/http/SimpleHTTPGetProcessor.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/a5764609/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
index dec9d03..d74793a 100644
--- a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
@@ -40,7 +40,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
+public class SimpleHTTPGetProcessor implements StreamsProcessor {
private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
@@ -152,7 +152,7 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
try {
response = httpclient.execute(httpget);
HttpEntity entity = response.getEntity();
- // TODO: handle rate-limiting
+ // TODO: handle retry
if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
entityString = EntityUtils.toString(entity);
}
@@ -181,7 +181,7 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
- entry.setDocument(extensionEntity);
+ entry.setDocument(rootDocument);
result.add(entry);
[06/32] incubator-streams git commit: committing PeoplePattern
processor
Posted by sb...@apache.org.
committing PeoplePattern processor
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/9a575322
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/9a575322
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/9a575322
Branch: refs/heads/STREAMS-212
Commit: 9a575322231e4a8a69c56f06da743ffe3211ccb4
Parents: d62061d
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Oct 12 20:20:52 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Oct 12 20:20:52 2014 -0500
----------------------------------------------------------------------
streams-contrib/pom.xml | 5 +-
.../streams-processor-peoplepattern/pom.xml | 138 +++++++++++++++++++
.../peoplepattern/AccountTypeProcessor.java | 76 ++++++++++
.../peoplepattern/DemographicsProcessor.java | 77 +++++++++++
.../streams/peoplepattern/AccountType.json | 27 ++++
.../streams/peoplepattern/Demographics.json | 60 ++++++++
.../resources/templates/peoplepatternactor.json | 25 ++++
7 files changed, 406 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
index e290466..fcec297 100644
--- a/streams-contrib/pom.xml
+++ b/streams-contrib/pom.xml
@@ -44,17 +44,18 @@
<module>streams-persist-hdfs</module>
<module>streams-persist-kafka</module>
<module>streams-persist-mongo</module>
- <module>streams-amazon-aws</module>
+ <module>streams-amazon-aws</module>
<!--<module>streams-processor-lucene</module>-->
<!--<module>streams-processor-tika</module>-->
- <module>streams-provider-instagram</module>
<module>streams-processor-jackson</module>
<module>streams-processor-json</module>
<module>streams-processor-urls</module>
+ <module>streams-processor-peoplepattern</module>
<module>streams-provider-datasift</module>
<module>streams-provider-facebook</module>
<module>streams-provider-google</module>
<module>streams-provider-gnip</module>
+ <module>streams-provider-instagram</module>
<module>streams-provider-moreover</module>
<module>streams-provider-twitter</module>
<module>streams-provider-sysomos</module>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/pom.xml b/streams-contrib/streams-processor-peoplepattern/pom.xml
new file mode 100644
index 0000000..b810200
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/pom.xml
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>streams-processor-peoplepattern</artifactId>
+ <version>0.1-SNAPSHOT</version>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-contrib</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-http</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo/**/*.java</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-source-jaxb2</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jaxb2</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema/org/apache/streams/peoplepattern</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>org.apache.streams.peoplepattern</targetPackage>
+ <useLongIntegers>true</useLongIntegers>
+ <useJodaDates>true</useJodaDates>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
new file mode 100644
index 0000000..d180b7f
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.peoplepattern;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Maps;
+import org.apache.streams.components.http.HttpConfigurator;
+import org.apache.streams.components.http.HttpProcessorConfiguration;
+import org.apache.streams.components.http.processor.SimpleHTTPGetProcessor;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.data.util.ExtensionUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * Enrich actor with demographics
+ */
+public class AccountTypeProcessor extends SimpleHTTPGetProcessor {
+
+ private final static String STREAMS_ID = "AccountTypeProcessor";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(AccountTypeProcessor.class);
+
+ public AccountTypeProcessor() {
+ this(HttpConfigurator.detectProcessorConfiguration(StreamsConfigurator.config.getConfig("peoplepattern")));
+ }
+
+ public AccountTypeProcessor(HttpProcessorConfiguration peoplePatternConfiguration) {
+ super(peoplePatternConfiguration);
+ LOGGER.info("creating AccountTypeProcessor");
+ configuration.setProtocol("https");
+ configuration.setHostname("api.peoplepattern.com");
+ configuration.setResourcePath("/v0.2/account_type/");
+ configuration.setEntity(HttpProcessorConfiguration.Entity.ACTOR);
+ configuration.setExtension("account_type");
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ @Override
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+ Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
+ //Actor actor = mapper.convertValue(entry.getDocument(), Actor.class);
+ Actor actor = activity.getActor();
+ ObjectNode actorObjectNode = mapper.convertValue(actor, ObjectNode.class);
+ String username = (String) ExtensionUtil.getExtension(actorObjectNode, "screenName");
+ Map<String, String> params = Maps.newHashMap();
+ params.put("id", actor.getId());
+ params.put("name", actor.getDisplayName());
+ params.put("username", username);
+ params.put("description", actor.getSummary());
+ return params;
+ }
+};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
new file mode 100644
index 0000000..6ffbb9b
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.peoplepattern;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Maps;
+import org.apache.streams.components.http.HttpConfigurator;
+import org.apache.streams.components.http.HttpProcessorConfiguration;
+import org.apache.streams.components.http.processor.SimpleHTTPGetProcessor;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.data.util.ExtensionUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.Actor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * Enrich actor with demographics
+ */
+public class DemographicsProcessor extends SimpleHTTPGetProcessor {
+
+ public final static String STREAMS_ID = "DemographicsProcessor";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(DemographicsProcessor.class);
+
+ public DemographicsProcessor() {
+ this(HttpConfigurator.detectProcessorConfiguration(StreamsConfigurator.config.getConfig("peoplepattern")));
+ }
+
+ public DemographicsProcessor(HttpProcessorConfiguration peoplePatternConfiguration) {
+ super(peoplePatternConfiguration);
+ LOGGER.info("creating DemographicsProcessor");
+ configuration.setProtocol("https");
+ configuration.setHostname("api.peoplepattern.com");
+ configuration.setResourcePath("/v0.2/demographics/");
+ configuration.setEntity(HttpProcessorConfiguration.Entity.ACTOR);
+ configuration.setExtension("demographics");
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ @Override
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+ Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
+ //Actor actor = mapper.convertValue(entry.getDocument(), Actor.class);
+ Actor actor = activity.getActor();
+ ObjectNode actorObjectNode = mapper.convertValue(actor, ObjectNode.class);
+ String username = (String) ExtensionUtil.getExtension(actorObjectNode, "screenName");
+ Map<String, String> params = Maps.newHashMap();
+ params.put("id", actor.getId());
+ params.put("name", actor.getDisplayName());
+ params.put("username", username);
+ params.put("description", actor.getSummary());
+ return params;
+ }
+
+};
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json
new file mode 100644
index 0000000..5656b44
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json
@@ -0,0 +1,27 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType": "org.apache.streams.peoplepattern.AccountType",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "prediction" : {
+ "type" : "string",
+ "enum" : [
+ "person",
+ "organization",
+ "entertainment",
+ "adult",
+ "spam",
+ "no-prediction"
+ ],
+ "default": "no-prediction"
+ },
+ "score": {
+ "type": "number"
+ },
+ "id": {
+ "type": "string"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json
new file mode 100644
index 0000000..d1f64d8
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json
@@ -0,0 +1,60 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType": "org.apache.streams.peoplepattern.Demographics",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "age": {
+ "type": "object",
+ "properties": {
+ "prediction": {
+ "type": "integer",
+ "default": 1990
+ },
+ "score": {
+ "type": "number"
+ }
+
+ }
+ },
+ "gender" : {
+ "type": "object",
+ "properties": {
+ "prediction": {
+ "type": "string",
+ "enum": [
+ "male",
+ "female",
+ "no-prediction"
+ ],
+ "default": "no-prediction"
+ },
+ "score": {
+ "type": "number"
+ }
+ }
+ },
+ "race" : {
+ "type": "object",
+ "properties": {
+ "prediction": {
+ "type": "string",
+ "enum": [
+ "black",
+ "east-asian",
+ "hispanic",
+ "middle-eastern",
+ "south-asian",
+ "white",
+ "no-prediction"
+ ],
+ "default": "no-prediction"
+ },
+ "score": {
+ "type": "number"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/9a575322/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json b/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json
new file mode 100644
index 0000000..9a24c5c
--- /dev/null
+++ b/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json
@@ -0,0 +1,25 @@
+{
+ "order": 20,
+ "template": "*activity*",
+ "settings": {},
+ "mappings": {
+ "activity": {
+ "properties": {
+ "actor": {
+ "properties": {
+ "extensions": {
+ "properties": {
+ "account_type": {
+ "type": "nested"
+ },
+ "demographics": {
+ "type": "nested"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
[07/32] incubator-streams git commit: removing validation,
as jackson does not contain a default implementation until 2.5 release
Posted by sb...@apache.org.
removing validation, as jackson does not contain a default implementation until 2.5 release
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ad5f90cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ad5f90cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ad5f90cc
Branch: refs/heads/STREAMS-212
Commit: ad5f90cc115c40e0ef480a85957bbb4c98157cf6
Parents: 9a57532
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Oct 12 20:21:22 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Oct 12 20:21:22 2014 -0500
----------------------------------------------------------------------
streams-components/streams-http/pom.xml | 6 ------
.../components/http/processor/SimpleHTTPGetProcessor.java | 6 ------
.../components/http/provider/SimpleHTTPGetProvider.java | 3 ---
3 files changed, 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ad5f90cc/streams-components/streams-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/pom.xml b/streams-components/streams-http/pom.xml
index 9c2b079..39a4faa 100644
--- a/streams-components/streams-http/pom.xml
+++ b/streams-components/streams-http/pom.xml
@@ -38,12 +38,6 @@
<artifactId>jsonschema2pojo-core</artifactId>
<type>jar</type>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ad5f90cc/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
index c2bfef6..0d17cc6 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
@@ -3,7 +3,6 @@ package org.apache.streams.components.http.processor;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -25,8 +24,6 @@ import org.apache.streams.jackson.StreamsJacksonMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.validation.Validation;
-import javax.validation.ValidatorFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -199,9 +196,6 @@ public class SimpleHTTPGetProcessor implements StreamsProcessor {
@Override
public void prepare(Object configurationObject) {
- ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
- Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
-
mapper = StreamsJacksonMapper.getInstance();
uriBuilder = new URIBuilder()
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ad5f90cc/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
index 622225a..36084c7 100644
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
@@ -89,9 +89,6 @@ public class SimpleHTTPGetProvider implements StreamsProvider {
@Override
public void prepare(Object configurationObject) {
-// ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
-// Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
-
mapper = StreamsJacksonMapper.getInstance();
uriBuilder = new URIBuilder()
[15/32] incubator-streams git commit: Fixed merge error
Posted by sb...@apache.org.
Fixed merge error
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/79eadea2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/79eadea2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/79eadea2
Branch: refs/heads/STREAMS-212
Commit: 79eadea2453610075236e1fed13c215bee474592
Parents: 377fb89
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Oct 31 17:00:47 2014 -0500
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Oct 31 17:00:47 2014 -0500
----------------------------------------------------------------------
.../java/org/apache/streams/local/tasks/StreamsProcessorTask.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/79eadea2/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index 33c5827..b6ab498 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -127,6 +127,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
if(output != null) {
for(StreamsDatum outDatum : output) {
super.addToOutgoingQueue(outDatum);
+ this.counter.incrementEmittedCount();
statusCounter.incrementStatus(DatumStatus.SUCCESS);
}
}
[27/32] incubator-streams git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-streams
Posted by sb...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-streams
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/01716fe3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/01716fe3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/01716fe3
Branch: refs/heads/STREAMS-212
Commit: 01716fe31d6e7507b5a1aaec7e7060de0371c68b
Parents: 660f0c1 ab9c696
Author: Ryan Ebanks <ry...@gmail.com>
Authored: Fri Nov 7 13:13:50 2014 -0600
Committer: Ryan Ebanks <ry...@gmail.com>
Committed: Fri Nov 7 13:13:50 2014 -0600
----------------------------------------------------------------------
pom.xml | 58 ++++
streams-components/pom.xml | 62 +++++
streams-components/streams-http/README.md | 16 ++
streams-components/streams-http/pom.xml | 153 +++++++++++
.../components/http/HttpConfigurator.java | 62 +++++
.../http/processor/SimpleHTTPGetProcessor.java | 268 +++++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 230 ++++++++++++++++
.../components/http/HttpConfiguration.json | 50 ++++
.../http/HttpProcessorConfiguration.json | 28 ++
.../http/HttpProviderConfiguration.json | 18 ++
streams-contrib/pom.xml | 5 +-
streams-contrib/streams-persist-console/pom.xml | 34 +++
.../streams/console/ConsolePersistReader.java | 11 +-
.../streams/console/ConsolePersistWriter.java | 11 +-
.../streams-processor-peoplepattern/pom.xml | 138 ++++++++++
.../peoplepattern/AccountTypeProcessor.java | 75 ++++++
.../peoplepattern/DemographicsProcessor.java | 76 ++++++
.../streams/peoplepattern/AccountType.json | 27 ++
.../streams/peoplepattern/Demographics.json | 60 +++++
.../resources/templates/peoplepatternactor.json | 25 ++
.../api/FacebookPostActivitySerializer.java | 1 -
.../streams-provider-twitter/pom.xml | 5 +
.../processor/TwitterUrlApiProcessor.java | 73 +++++
.../provider/TwitterTimelineProvider.java | 7 +-
streams-pojo-extensions/pom.xml | 64 +++++
.../apache/streams/data/util/ExtensionUtil.java | 108 ++++++++
.../apache/streams/data/util/ActivityUtil.java | 14 +-
.../jackson/StreamsDateTimeDeserializer.java | 23 +-
.../streams/jackson/StreamsJacksonMapper.java | 20 ++
.../streams/jackson/StreamsJacksonModule.java | 9 +
.../org/apache/streams/pojo/json/activity.json | 3 +-
.../org/apache/streams/pojo/json/object.json | 2 +-
32 files changed, 1713 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
[05/32] incubator-streams git commit: introduced a
SimpleHTTPGetProvider reorganized configuration added access_token support
added http basic auth support
Posted by sb...@apache.org.
introduced a SimpleHTTPGetProvider
reorganized configuration
added access_token support
added http basic auth support
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d62061de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d62061de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d62061de
Branch: refs/heads/STREAMS-212
Commit: d62061dec8628484e66d3494e02f1f65efafda41
Parents: a576460
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Sun Oct 12 17:49:01 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Sun Oct 12 17:49:01 2014 -0500
----------------------------------------------------------------------
streams-components/pom.xml | 2 +-
streams-components/streams-http/README.md | 16 ++
streams-components/streams-http/pom.xml | 159 +++++++++++++
.../components/http/HttpConfigurator.java | 62 +++++
.../http/processor/SimpleHTTPGetProcessor.java | 230 +++++++++++++++++++
.../http/provider/SimpleHTTPGetProvider.java | 215 +++++++++++++++++
.../components/http/HttpConfiguration.json | 50 ++++
.../http/HttpProcessorConfiguration.json | 28 +++
.../http/HttpProviderConfiguration.json | 18 ++
.../streams-processor-http/README.md | 16 --
.../streams-processor-http/pom.xml | 154 -------------
.../components/http/HttpConfigurator.java | 53 -----
.../components/http/SimpleHTTPGetProcessor.java | 218 ------------------
.../HttpProcessorConfiguration.json | 47 ----
14 files changed, 779 insertions(+), 489 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/pom.xml b/streams-components/pom.xml
index 26384b1..9942e14 100644
--- a/streams-components/pom.xml
+++ b/streams-components/pom.xml
@@ -37,7 +37,7 @@
</properties>
<modules>
- <module>streams-processor-http</module>
+ <module>streams-http</module>
</modules>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/README.md
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/README.md b/streams-components/streams-http/README.md
new file mode 100644
index 0000000..62dd4c1
--- /dev/null
+++ b/streams-components/streams-http/README.md
@@ -0,0 +1,16 @@
+streams-processor-http
+=====================
+
+Hit an http endpoint and place the result in extensions
+
+Example SimpleHTTPGetProcessor configuration:
+
+ "http": {
+ "protocol": "http",
+ "hostname": "urls.api.twitter.com",
+ "port": 9300,
+ "resourceUri": "1/urls/count.json"
+ }
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/pom.xml b/streams-components/streams-http/pom.xml
new file mode 100644
index 0000000..9c2b079
--- /dev/null
+++ b/streams-components/streams-http/pom.xml
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-components</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>streams-http</artifactId>
+
+ <name>streams-http</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-core</artifactId>
+ <type>jar</type>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.validation</groupId>
+ <artifactId>validation-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo-extensions</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.3.5</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-source-jaxb2</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jaxb2</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>org.apache.streams.http</targetPackage>
+ <useLongIntegers>true</useLongIntegers>
+ <useJodaDates>true</useJodaDates>
+ <includeJsr303Annotations>true</includeJsr303Annotations>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
new file mode 100644
index 0000000..900831f
--- /dev/null
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.components.http;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigRenderOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
+ */
+public class HttpConfigurator {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(HttpConfigurator.class);
+
+ private final static ObjectMapper mapper = new ObjectMapper();
+
+ public static HttpProviderConfiguration detectProviderConfiguration(Config config) {
+
+ HttpProviderConfiguration httpProviderConfiguration = null;
+
+ try {
+ httpProviderConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), HttpProviderConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse http configuration", e.getMessage());
+ }
+ return httpProviderConfiguration;
+ }
+
+ public static HttpProcessorConfiguration detectProcessorConfiguration(Config config) {
+
+ HttpProcessorConfiguration httpProcessorConfiguration = null;
+
+ try {
+ httpProcessorConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), HttpProcessorConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse http configuration", e.getMessage());
+ }
+ return httpProcessorConfiguration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
new file mode 100644
index 0000000..c2bfef6
--- /dev/null
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
@@ -0,0 +1,230 @@
+package org.apache.streams.components.http.processor;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.streams.components.http.HttpConfigurator;
+import org.apache.streams.components.http.HttpProcessorConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.data.util.ExtensionUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.Validation;
+import javax.validation.ValidatorFactory;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Processor retrieves contents from an known url and stores the resulting object in an extension field
+ */
+public class SimpleHTTPGetProcessor implements StreamsProcessor {
+
+ private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
+
+ // from root config id
+ private final static String EXTENSION = "account_type";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPGetProcessor.class);
+
+ protected ObjectMapper mapper;
+
+ protected URIBuilder uriBuilder;
+
+ protected CloseableHttpClient httpclient;
+
+ protected HttpProcessorConfiguration configuration;
+
+ protected String authHeader;
+//
+// // authorized only
+// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
+// //private String authHeader;
+//
+ public SimpleHTTPGetProcessor() {
+ this(HttpConfigurator.detectProcessorConfiguration(StreamsConfigurator.config.getConfig("http")));
+ }
+
+ public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
+ LOGGER.info("creating SimpleHTTPGetProcessor");
+ LOGGER.info(processorConfiguration.toString());
+ this.configuration = processorConfiguration;
+ }
+
+
+ /**
+ Override this to store a result other than exact json representation of response
+ */
+ protected ObjectNode prepareExtensionFragment(String entityString) {
+
+ try {
+ return mapper.readValue(entityString, ObjectNode.class);
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ }
+ }
+
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode getRootDocument(StreamsDatum datum) {
+
+ try {
+ String json = datum.getDocument() instanceof String ?
+ (String) datum.getDocument() :
+ mapper.writeValueAsString(datum.getDocument());
+ return mapper.readValue(json, ObjectNode.class);
+ } catch (JsonProcessingException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ return null;
+ }
+
+ }
+ /**
+ Override this to place result in non-standard location on document
+ */
+ protected ObjectNode getEntityToExtend(ObjectNode rootDocument) {
+
+ if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
+ return rootDocument;
+ else
+ return (ObjectNode) rootDocument.get(this.configuration.getEntity().toString());
+
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+
+ List<StreamsDatum> result = Lists.newArrayList();
+
+ ObjectNode rootDocument = getRootDocument(entry);
+
+ Map<String, String> params = prepareParams(entry);
+
+ URI uri;
+ for( Map.Entry<String,String> param : params.entrySet()) {
+ uriBuilder = uriBuilder.setParameter(param.getKey(), param.getValue());
+ }
+ try {
+ uri = uriBuilder.build();
+ } catch (URISyntaxException e) {
+ LOGGER.error("URI error {}", uriBuilder.toString());
+ return result;
+ }
+
+ HttpGet httpget = prepareHttpGet(uri);
+
+ CloseableHttpResponse response = null;
+
+ String entityString = null;
+ try {
+ response = httpclient.execute(httpget);
+ HttpEntity entity = response.getEntity();
+ // TODO: handle retry
+ if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
+ entityString = EntityUtils.toString(entity);
+ }
+ } catch (IOException e) {
+ LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, e.getMessage());
+ return result;
+ } finally {
+ try {
+ response.close();
+ } catch (IOException e) {}
+ try {
+ httpclient.close();
+ } catch (IOException e) {}
+ }
+
+ if( entityString == null )
+ return result;
+
+ LOGGER.debug(entityString);
+
+ ObjectNode extensionFragment = prepareExtensionFragment(entityString);
+
+ ObjectNode extensionEntity = getEntityToExtend(rootDocument);
+
+ ExtensionUtil.ensureExtensions(extensionEntity);
+
+ ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
+
+ entry.setDocument(rootDocument);
+
+ result.add(entry);
+
+ return result;
+
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+
+ return Maps.newHashMap();
+ }
+
+
+ public HttpGet prepareHttpGet(URI uri) {
+ HttpGet httpget = new HttpGet(uri);
+ httpget.addHeader("content-type", this.configuration.getContentType());
+ if( !Strings.isNullOrEmpty(authHeader))
+ httpget.addHeader("Authorization", String.format("Basic %s", authHeader));
+ return httpget;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
+ Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
+
+ mapper = StreamsJacksonMapper.getInstance();
+
+ uriBuilder = new URIBuilder()
+ .setScheme(this.configuration.getProtocol())
+ .setHost(this.configuration.getHostname())
+ .setPath(this.configuration.getResourcePath());
+
+ if( !Strings.isNullOrEmpty(configuration.getAccessToken()) )
+ uriBuilder = uriBuilder.addParameter("access_token", configuration.getAccessToken());
+ if( !Strings.isNullOrEmpty(configuration.getUsername())
+ && !Strings.isNullOrEmpty(configuration.getPassword())) {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append(configuration.getUsername());
+ stringBuilder.append(":");
+ stringBuilder.append(configuration.getPassword());
+ String string = stringBuilder.toString();
+ authHeader = Base64.encodeBase64String(string.getBytes());
+ }
+ httpclient = HttpClients.createDefault();
+ }
+
+ @Override
+ public void cleanUp() {
+ LOGGER.info("shutting down SimpleHTTPGetProcessor");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
new file mode 100644
index 0000000..622225a
--- /dev/null
+++ b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHTTPGetProvider.java
@@ -0,0 +1,215 @@
+package org.apache.streams.components.http.provider;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.streams.components.http.HttpConfigurator;
+import org.apache.streams.components.http.HttpProviderConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Provider retrieves contents from an known set of urls and passes all resulting objects downstream
+ */
+public class SimpleHTTPGetProvider implements StreamsProvider {
+
+ private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
+
+ // from root config id
+ private final static String EXTENSION = "account_type";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPGetProvider.class);
+
+ protected ObjectMapper mapper;
+
+ protected URIBuilder uriBuilder;
+
+ protected CloseableHttpClient httpclient;
+
+ protected HttpProviderConfiguration configuration;
+
+ protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<StreamsDatum>();
+
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
+
+ // // authorized only
+// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
+// //private String authHeader;
+//
+ public SimpleHTTPGetProvider() {
+ this(HttpConfigurator.detectProviderConfiguration(StreamsConfigurator.config.getConfig("http")));
+ }
+
+ public SimpleHTTPGetProvider(HttpProviderConfiguration providerConfiguration) {
+ LOGGER.info("creating SimpleHTTPGetProvider");
+ LOGGER.info(providerConfiguration.toString());
+ this.configuration = providerConfiguration;
+ }
+
+ /**
+ Override this to add parameters to the request
+ */
+ protected Map<String, String> prepareParams(StreamsDatum entry) {
+
+ return Maps.newHashMap();
+ }
+
+ public HttpGet prepareHttpGet(URI uri) {
+ HttpGet httpget = new HttpGet(uri);
+ httpget.addHeader("content-type", this.configuration.getContentType());
+ return httpget;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+// ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
+// Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
+
+ mapper = StreamsJacksonMapper.getInstance();
+
+ uriBuilder = new URIBuilder()
+ .setScheme(this.configuration.getProtocol())
+ .setHost(this.configuration.getHostname())
+ .setPath(this.configuration.getResourcePath());
+
+ httpclient = HttpClients.createDefault();
+ }
+
+ @Override
+ public void cleanUp() {
+
+ LOGGER.info("shutting down SimpleHTTPGetProvider");
+ try {
+ httpclient.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ httpclient.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ httpclient = null;
+ }
+ }
+ }
+
+ @Override
+ public void startStream() {
+
+ }
+
+ @Override
+ public StreamsResultSet readCurrent() {
+ StreamsResultSet current;
+
+ uriBuilder = uriBuilder.setPath(
+ Joiner.on("/").skipNulls().join(uriBuilder.getPath(), configuration.getResource(), configuration.getResourcePostfix())
+ );
+
+ URI uri;
+ try {
+ uri = uriBuilder.build();
+ } catch (URISyntaxException e) {
+ uri = null;
+ }
+
+ List<ObjectNode> results = executeGet(uri);
+
+ lock.writeLock().lock();
+
+ for( ObjectNode item : results ) {
+ providerQueue.add(new StreamsDatum(item, item.get("id").asText(), new DateTime(item.get("timestamp").asText())));
+ }
+
+ LOGGER.debug("Creating new result set for {} items", providerQueue.size());
+ current = new StreamsResultSet(providerQueue);
+
+ return current;
+ }
+
+ protected List<ObjectNode> executeGet(URI uri) {
+
+ Preconditions.checkNotNull(uri);
+
+ List<ObjectNode> results = new ArrayList<>();
+
+ HttpGet httpget = prepareHttpGet(uri);
+
+ CloseableHttpResponse response = null;
+
+ String entityString = null;
+ try {
+ response = httpclient.execute(httpget);
+ HttpEntity entity = response.getEntity();
+ // TODO: handle retry
+ if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
+ entityString = EntityUtils.toString(entity);
+ if( !entityString.equals("{}") && !entityString.equals("[]") ) {
+ JsonNode jsonNode = mapper.readValue(entityString, JsonNode.class);
+ if (jsonNode != null && jsonNode instanceof ObjectNode ) {
+
+ results.add((ObjectNode) jsonNode);
+ } else if (jsonNode != null && jsonNode instanceof ArrayNode) {
+ ArrayNode arrayNode = (ArrayNode) jsonNode;
+ Iterator<JsonNode> iterator = arrayNode.elements();
+ while (iterator.hasNext()) {
+ ObjectNode element = (ObjectNode) iterator.next();
+
+ results.add(element);
+ }
+ }
+ }
+ }
+ } catch (IOException e) {
+ LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, e.getMessage());
+ } finally {
+ try {
+ response.close();
+ } catch (IOException e) {}
+ }
+ return results;
+ }
+
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ return null;
+ }
+
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ return null;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json
new file mode 100644
index 0000000..b4dc243
--- /dev/null
+++ b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json
@@ -0,0 +1,50 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.components.http.HttpConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "protocol": {
+ "type": "string",
+ "description": "Protocol",
+ "default": "http"
+ },
+ "hostname": {
+ "type": "string",
+ "description": "Hostname",
+ "required" : true
+ },
+ "port": {
+ "type": "integer",
+ "description": "Port",
+ "default": 80
+ },
+ "resourcePath": {
+ "type": "string",
+ "description": "Resource Path",
+ "required" : true
+ },
+ "content-type": {
+ "type": "string",
+ "description": "Resource content-type",
+ "required" : true,
+ "default": "application/json"
+ },
+ "access_token": {
+ "type": "string",
+ "description": "Known Access Token",
+ "required" : false
+ },
+ "username": {
+ "type": "string",
+ "description": "Basic Auth Username",
+ "required" : false
+ },
+ "password": {
+ "type": "string",
+ "description": "Basic Auth Password",
+ "required" : false
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json
new file mode 100644
index 0000000..32e4c23
--- /dev/null
+++ b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json
@@ -0,0 +1,28 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.components.http.HttpProcessorConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "extends": { "$ref": "HttpConfiguration.json" },
+ "properties": {
+ "entity": {
+ "type": "string",
+ "description": "Entity to extend",
+ "enum": [ "activity", "actor", "object", "target" ],
+ "required" : true,
+ "default": "activity"
+ },
+ "extension": {
+ "type": "string",
+ "description": "Extension identifier",
+ "required" : true
+ },
+ "urlField": {
+ "type": "string",
+ "description": "Field where url is located",
+ "required" : true,
+ "default": "url"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json
new file mode 100644
index 0000000..2c135d9
--- /dev/null
+++ b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json
@@ -0,0 +1,18 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.components.http.HttpProviderConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "extends": { "$ref": "HttpConfiguration.json" },
+ "properties": {
+ "resource": {
+ "type": "string",
+ "required" : false
+ },
+ "resourcePostfix": {
+ "type": "string",
+ "required" : false
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/README.md
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/README.md b/streams-components/streams-processor-http/README.md
deleted file mode 100644
index 62dd4c1..0000000
--- a/streams-components/streams-processor-http/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-streams-processor-http
-=====================
-
-Hit an http endpoint and place the result in extensions
-
-Example SimpleHTTPGetProcessor configuration:
-
- "http": {
- "protocol": "http",
- "hostname": "urls.api.twitter.com",
- "port": 9300,
- "resourceUri": "1/urls/count.json"
- }
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/pom.xml
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/pom.xml b/streams-components/streams-processor-http/pom.xml
deleted file mode 100644
index d9215ad..0000000
--- a/streams-components/streams-processor-http/pom.xml
+++ /dev/null
@@ -1,154 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-components</artifactId>
- <version>0.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>streams-processor-http</artifactId>
-
- <name>streams-processor-http</name>
-
- <dependencies>
-
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.3.5</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.8</version>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jsonschema2pojo</source>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>add-source-jaxb2</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jaxb2</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <configuration>
- <addCompileSourceRoot>true</addCompileSourceRoot>
- <generateBuilders>true</generateBuilders>
- <sourcePaths>
- <sourcePath>src/main/jsonschema</sourcePath>
- </sourcePaths>
- <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>org.apache.streams.http</targetPackage>
- <useLongIntegers>true</useLongIntegers>
- <useJodaDates>true</useJodaDates>
- <includeJsr303Annotations>true</includeJsr303Annotations>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
deleted file mode 100644
index 36801b8..0000000
--- a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.components.http;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
- */
-public class HttpConfigurator {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(HttpConfigurator.class);
-
- private final static ObjectMapper mapper = new ObjectMapper();
-
- public static HttpProcessorConfiguration detectConfiguration(Config config) {
-
- HttpProcessorConfiguration httpProcessorConfiguration = null;
-
- try {
- httpProcessorConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), HttpProcessorConfiguration.class);
- } catch (Exception e) {
- e.printStackTrace();
- LOGGER.warn("Could not parse http configuration", e.getMessage());
- }
- return httpProcessorConfiguration;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
deleted file mode 100644
index d74793a..0000000
--- a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
+++ /dev/null
@@ -1,218 +0,0 @@
-package org.apache.streams.components.http;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigValue;
-import org.apache.commons.io.IOUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.data.util.ActivityUtil;
-import org.apache.streams.data.util.ExtensionUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.validation.Validation;
-import javax.validation.ValidatorFactory;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-public class SimpleHTTPGetProcessor implements StreamsProcessor {
-
- private final static String STREAMS_ID = "SimpleHTTPGetProcessor";
-
- // from root config id
- private final static String EXTENSION = "account_type";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPGetProcessor.class);
-
- protected ObjectMapper mapper;
-
- protected URIBuilder uriBuilder;
-
- protected CloseableHttpClient httpclient;
-
- protected HttpProcessorConfiguration configuration;
-//
-// // authorized only
-// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
-// //private String authHeader;
-//
- public SimpleHTTPGetProcessor() {
- LOGGER.info("creating SimpleHTTPGetProcessor");
- this.configuration = HttpConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("http"));
- }
-
- public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
- LOGGER.info("creating SimpleHTTPGetProcessor");
- LOGGER.info(processorConfiguration.toString());
- this.configuration = processorConfiguration;
- }
-
- /**
- Override this to add parameters to the request
- */
- protected Map<String, String> prepareParams(StreamsDatum entry) {
-
- return Maps.newHashMap();
- }
-
- /**
- Override this to store a result other than exact json representation of response
- */
- protected ObjectNode prepareExtensionFragment(String entityString) {
-
- try {
- return mapper.readValue(entityString, ObjectNode.class);
- } catch (IOException e) {
- LOGGER.warn(e.getMessage());
- return null;
- }
- }
-
- /**
- Override this to place result in non-standard location on document
- */
- protected ObjectNode getRootDocument(StreamsDatum datum) {
-
- try {
- String json = datum.getDocument() instanceof String ?
- (String) datum.getDocument() :
- mapper.writeValueAsString(datum.getDocument());
- return mapper.readValue(json, ObjectNode.class);
- } catch (JsonProcessingException e) {
- LOGGER.warn(e.getMessage());
- return null;
- } catch (IOException e) {
- LOGGER.warn(e.getMessage());
- return null;
- }
-
- }
- /**
- Override this to place result in non-standard location on document
- */
- protected ObjectNode getEntityToExtend(ObjectNode rootDocument) {
-
- if( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY))
- return rootDocument;
- else
- return (ObjectNode) rootDocument.get(this.configuration.getEntity().toString());
-
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = Lists.newArrayList();
-
- ObjectNode rootDocument = getRootDocument(entry);
-
- Map<String, String> params = prepareParams(entry);
-
- URI uri;
- for( Map.Entry<String,String> param : params.entrySet()) {
- uriBuilder = uriBuilder.setParameter(param.getKey(), param.getValue());
- }
- try {
- uri = uriBuilder.build();
- } catch (URISyntaxException e) {
- LOGGER.error("URI error {}", uriBuilder.toString());
- return result;
- }
-
- HttpGet httpget = prepareHttpGet(uri);
-
- CloseableHttpResponse response = null;
-
- String entityString = null;
- try {
- response = httpclient.execute(httpget);
- HttpEntity entity = response.getEntity();
- // TODO: handle retry
- if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
- entityString = EntityUtils.toString(entity);
- }
- } catch (IOException e) {
- LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, e.getMessage());
- return result;
- } finally {
- try {
- response.close();
- } catch (IOException e) {}
- try {
- httpclient.close();
- } catch (IOException e) {}
- }
-
- if( entityString == null )
- return result;
-
- LOGGER.debug(entityString);
-
- ObjectNode extensionFragment = prepareExtensionFragment(entityString);
-
- ObjectNode extensionEntity = getEntityToExtend(rootDocument);
-
- ExtensionUtil.ensureExtensions(extensionEntity);
-
- ExtensionUtil.addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
-
- entry.setDocument(rootDocument);
-
- result.add(entry);
-
- return result;
-
- }
-
- public HttpGet prepareHttpGet(URI uri) {
- HttpGet httpget = new HttpGet(uri);
- httpget.addHeader("content-type", this.configuration.getContentType());
- return httpget;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
- Preconditions.checkArgument(factory.getValidator().validate(this.configuration, HttpProcessorConfiguration.class).size() == 0);
-
- mapper = StreamsJacksonMapper.getInstance();
-
- uriBuilder = new URIBuilder()
- .setScheme(this.configuration.getProtocol())
- .setHost(this.configuration.getHostname())
- .setPath(this.configuration.getResourceUri());
-
- httpclient = HttpClients.createDefault();
- }
-
- @Override
- public void cleanUp() {
- LOGGER.info("shutting down SimpleHTTPGetProcessor");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d62061de/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json b/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
deleted file mode 100644
index 40c3bcd..0000000
--- a/streams-components/streams-processor-http/src/main/jsonschema/org/apache/streams/elasticsearch/HttpProcessorConfiguration.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "javaType" : "org.apache.streams.components.http.HttpProcessorConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "Protocol",
- "default": "http"
- },
- "hostname": {
- "type": "string",
- "description": "Hostname",
- "required" : true
- },
- "port": {
- "type": "integer",
- "description": "Port",
- "default": 80
- },
- "resourceUri": {
- "type": "string",
- "description": "Resource URI",
- "required" : true
- },
- "content-type": {
- "type": "string",
- "description": "Resource URI",
- "required" : true,
- "default": "application/json"
- },
- "entity": {
- "type": "string",
- "description": "Entity to extend",
- "enum": [ "activity", "actor", "object", "target" ],
- "required" : true,
- "default": "activity"
- },
- "extension": {
- "type": "string",
- "description": "Extension identifier",
- "required" : true
- }
- }
-}
\ No newline at end of file