You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2015/03/31 17:00:46 UTC
[1/2] incubator-streams-examples git commit: basic implementation
(untested)
Repository: incubator-streams-examples
Updated Branches:
refs/heads/STREAMS-292 [created] 8b469d253
basic implementation (untested)
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/ded4a973
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/ded4a973
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/ded4a973
Branch: refs/heads/STREAMS-292
Commit: ded4a973a4cb13e4819eb490e33f26ddb8d423e1
Parents: a05d4c9
Author: sblackmon <sb...@apache.org>
Authored: Sun Feb 22 15:21:36 2015 -0600
Committer: sblackmon <sb...@apache.org>
Committed: Mon Feb 23 10:39:08 2015 -0600
----------------------------------------------------------------------
local/pom.xml | 4 +-
local/twitter-history-elasticsearch/README.md | 79 +++++
.../TwitterHistoryElasticsearch.png | Bin 0 -> 10508 bytes
local/twitter-history-elasticsearch/pom.xml | 296 +++++++++++++++++++
.../twitter/TwitterHistoryElasticsearch.java | 89 ++++++
...witterHistoryElasticsearchConfiguration.json | 13 +
.../resources/TwitterHistoryElasticsearch.dot | 21 ++
.../src/main/resources/application.conf | 23 ++
.../twitter-history-elasticsearch.png | Bin 0 -> 10508 bytes
9 files changed, 524 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/ded4a973/local/pom.xml
----------------------------------------------------------------------
diff --git a/local/pom.xml b/local/pom.xml
index 184d463..3b277d3 100644
--- a/local/pom.xml
+++ b/local/pom.xml
@@ -37,7 +37,9 @@
</properties>
<modules>
- <module>elasticsearch-reindex</module>
+
+ <module>twitter-history-elasticsearch</module>
+
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/ded4a973/local/twitter-history-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/README.md b/local/twitter-history-elasticsearch/README.md
new file mode 100644
index 0000000..649953b
--- /dev/null
+++ b/local/twitter-history-elasticsearch/README.md
@@ -0,0 +1,79 @@
+Apache Streams (incubating)
+Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------
+
+twitter-history-elasticsearch
+==============================
+
+Requirements:
+-------------
+ - Authorized Twitter API credentials
+ - A running ElasticSearch 1.0.0+ instance
+
+Description:
+------------
+Retrieves as many posts from a known list of users as twitter API allows.
+
+Converts them to activities, and writes them in activity format to Elasticsearch.
+
+Specification:
+-----------------
+
+[TwitterHistoryElasticsearch.dot](src/main/resources/TwitterHistoryElasticsearch.dot "TwitterHistoryElasticsearch.dot" )
+
+Diagram:
+-----------------
+
+
+
+Example Configuration:
+----------------------
+
+ twitter {
+ host = "api.twitter.com"
+ endpoint = "statuses/user_timeline"
+ oauth {
+ consumerKey = ""
+ consumerSecret = ""
+ accessToken = ""
+ accessTokenSecret = ""
+ }
+ info = [
+ "42232950"
+ "211620426"
+ ]
+ }
+ elasticsearch {
+ hosts = [
+ localhost
+ ]
+ port = 9300
+ clusterName = elasticsearch
+ index = userhistory_activity
+ type = activity
+ }
+
+In the Twitter section you should place all of your relevant authentication keys and whichever Twitter IDs you're looking to follow
+Twitter IDs can be converted from screennames at http://www.gettwitterid.com
+
+Running:
+--------
+
+You will need to run `./install_templates.sh` in the resources folder in order to apply the templates to your ES cluster
+
+ java -cp target/twitter-history-elasticsearch-0.1-SNAPSHOT.jar -Dconfig.file=application.conf org.apache.streams.twitter.example.TwitterHistoryElasticsearchActivity
+
+Note that you must modify src/main/resources/application.conf, and supply an absolute path to config.file
+
+Verification:
+-------------
+Open up http://localhost:9200/_plugin/head/ and confirm that the index you specified now contains has data
+
+Download https://github.com/w2ogroup/streams-examples/blob/master/twitter-history-elasticsearch/src/main/resources/reports/ActivityReport.json
+
+Open up http://localhost:9200/_plugin/marvel and from the folder icon in the top right hand corner click
+ Load -> Advanced -> Choose File and select the report you downloaded
+
+The gear on the top-right allows you to change the report index
+
+You should now see dashboards displaying metrics about your twitter activity
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/ded4a973/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png b/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png
new file mode 100644
index 0000000..d17e1cd
Binary files /dev/null and b/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/ded4a973/local/twitter-history-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/pom.xml b/local/twitter-history-elasticsearch/pom.xml
new file mode 100644
index 0000000..06855c1
--- /dev/null
+++ b/local/twitter-history-elasticsearch/pom.xml
@@ -0,0 +1,296 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-examples-local</artifactId>
+ <version>0.2-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>twitter-history-elasticsearch</artifactId>
+
+ <properties>
+ <elasticsearch.version>1.1.0</elasticsearch.version>
+ <lucene.version>4.7.2</lucene.version>
+ </properties>
+
+ <dependencies>
+ <!-- Test includes -->
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${lucene.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-codecs</artifactId>
+ <version>${lucene.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>1.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ <version>1.2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-util</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-runtime-local</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-provider-twitter</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-persist-elasticsearch</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>data</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
+ <!-- This binary runs with logback -->
+ <!-- Keep log4j out -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>1.3.1</version>
+ <executions>
+ <execution>
+ <id>enforce-banned-dependencies</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <bannedDependencies>
+ <excludes>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
+ <exclude>org.slf4j:slf4j-jcl</exclude>
+ <exclude>org.slf4j:slf4j-jdk14</exclude>
+ <exclude>org.log4j:log4j</exclude>
+ <exclude>commons-logging:commons-logging</exclude>
+ </excludes>
+ </bannedDependencies>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.build.finalName}</finalName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>**/logback.xml</exclude>
+ <exclude>**/log4j.properties</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.apache.streams.example.twitter.TwitterHistoryElasticsearch</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <version>0.4.1</version>
+ <configuration>
+ <addCompileSourceRoot>true</addCompileSourceRoot>
+ <generateBuilders>true</generateBuilders>
+ <sourcePaths>
+ <sourcePath>src/main/jsonschema</sourcePath>
+ </sourcePaths>
+ <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
+ <targetPackage>org.apache.streams.example.twitter</targetPackage>
+ <useJodaDates>false</useJodaDates>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/jsonschema2pojo</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <id>resource-dependencies</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeArtifactIds>streams-pojo</includeArtifactIds>
+ <includes>**/*.json</includes>
+ <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>2.12.4</version>
+ <executions>
+ <execution>
+ <id>integration-tests</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/ded4a973/local/twitter-history-elasticsearch/src/main/java/org/apache/streams/example/twitter/TwitterHistoryElasticsearch.java
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/main/java/org/apache/streams/example/twitter/TwitterHistoryElasticsearch.java b/local/twitter-history-elasticsearch/src/main/java/org/apache/streams/example/twitter/TwitterHistoryElasticsearch.java
new file mode 100644
index 0000000..2e1c886
--- /dev/null
+++ b/local/twitter-history-elasticsearch/src/main/java/org/apache/streams/example/twitter/TwitterHistoryElasticsearch.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.example.twitter;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.typesafe.config.Config;
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamBuilder;
+import org.apache.streams.elasticsearch.ElasticsearchConfigurator;
+import org.apache.streams.elasticsearch.ElasticsearchPersistWriter;
+import org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration;
+import org.apache.streams.local.builders.LocalStreamBuilder;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.twitter.TwitterStreamConfiguration;
+import org.apache.streams.twitter.TwitterUserInformationConfiguration;
+import org.apache.streams.twitter.processor.TwitterTypeConverter;
+import org.apache.streams.twitter.provider.TwitterConfigurator;
+import org.apache.streams.twitter.provider.TwitterTimelineProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Retrieves as many posts from a known list of users as twitter API allows.
+ *
+ * Converts them to activities, and writes them in activity format to Elasticsearch.
+ */
+
+public class TwitterHistoryElasticsearch implements Runnable {
+
+ public final static String STREAMS_ID = "TwitterHistoryElasticsearch";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(TwitterHistoryElasticsearch.class);
+
+ private static final ObjectMapper mapper = new ObjectMapper();
+
+ TwitterHistoryElasticsearchConfiguration config;
+
+ public TwitterHistoryElasticsearch() {
+ this(new ComponentConfigurator<>(TwitterHistoryElasticsearchConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig()));
+
+ }
+
+ public TwitterHistoryElasticsearch(TwitterHistoryElasticsearchConfiguration config) {
+ this.config = config;
+ }
+
+ public static void main(String[] args)
+ {
+ LOGGER.info(StreamsConfigurator.config.toString());
+
+ TwitterHistoryElasticsearch history = new TwitterHistoryElasticsearch();
+
+ new Thread(history).start();
+
+ }
+
+
+ public void run() {
+
+ TwitterTimelineProvider provider = new TwitterTimelineProvider(config.getTwitter(), ObjectNode.class);
+ TwitterTypeConverter converter = new TwitterTypeConverter(ObjectNode.class, Activity.class);
+ ElasticsearchPersistWriter writer = new ElasticsearchPersistWriter(config.getElasticsearch());
+
+ StreamBuilder builder = new LocalStreamBuilder(500);
+
+ builder.newPerpetualStream("provider", provider);
+ builder.addStreamsProcessor("converter", converter, 2, "provider");
+ builder.addStreamsPersistWriter("writer", writer, 1, "converter");
+ builder.start();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/ded4a973/local/twitter-history-elasticsearch/src/main/jsonschema/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/main/jsonschema/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json b/local/twitter-history-elasticsearch/src/main/jsonschema/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json
new file mode 100644
index 0000000..ea9b165
--- /dev/null
+++ b/local/twitter-history-elasticsearch/src/main/jsonschema/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json
@@ -0,0 +1,13 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "type": "object",
+ "javaType" : "org.apache.streams.example.twitter.TwitterHistoryElasticsearchConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "twitter": { "javaType": "org.apache.streams.twitter.TwitterUserInformationConfiguration", "type": "object", "required": true },
+ "elasticsearch": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration", "type": "object", "required": true }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/ded4a973/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot b/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot
new file mode 100644
index 0000000..e13af61
--- /dev/null
+++ b/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot
@@ -0,0 +1,21 @@
+digraph g {
+
+ //providers
+ TwitterTimelineProvider [label="TwitterTimelineProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java"];
+
+ //processors
+ TwitterTypeConverter [label="TwitterTypeConverter",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java"];
+
+ //persisters
+ ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"];
+
+ //data
+ es [label="es://{index}/{type}",shape=box];
+
+ //stream
+ TwitterTimelineProvider -> TwitterTypeConverter [label="ObjectNode"];
+ TwitterTypeConverter -> ElasticsearchPersistWriter [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+
+ ElasticsearchPersistWriter -> es [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/ded4a973/local/twitter-history-elasticsearch/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/main/resources/application.conf b/local/twitter-history-elasticsearch/src/main/resources/application.conf
new file mode 100644
index 0000000..c74b54b
--- /dev/null
+++ b/local/twitter-history-elasticsearch/src/main/resources/application.conf
@@ -0,0 +1,23 @@
+twitter {
+ host = "api.twitter.com"
+ endpoint = "statuses/user_timeline"
+ oauth {
+ consumerKey = ""
+ consumerSecret = ""
+ accessToken = ""
+ accessTokenSecret = ""
+ }
+ info = [
+ "42232950"
+ "211620426"
+ ]
+}
+elasticsearch {
+ hosts = [
+ localhost
+ ]
+ port = 9300
+ clusterName = elasticsearch
+ index = twitterhistory_activity
+ type = activity
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/ded4a973/local/twitter-history-elasticsearch/twitter-history-elasticsearch.png
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/twitter-history-elasticsearch.png b/local/twitter-history-elasticsearch/twitter-history-elasticsearch.png
new file mode 100644
index 0000000..d17e1cd
Binary files /dev/null and b/local/twitter-history-elasticsearch/twitter-history-elasticsearch.png differ
[2/2] incubator-streams-examples git commit: clean up documentation
switch to ActivityConverterProcessor confirmed working
Posted by sb...@apache.org.
clean up documentation
switch to ActivityConverterProcessor
confirmed working
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/8b469d25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/8b469d25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/8b469d25
Branch: refs/heads/STREAMS-292
Commit: 8b469d25358814cb474361ca8dd009264a3d753b
Parents: ded4a97
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Tue Mar 31 10:00:46 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Tue Mar 31 10:00:46 2015 -0500
----------------------------------------------------------------------
local/twitter-history-elasticsearch/README.md | 29 ++++++------
.../TwitterHistoryElasticsearch.png | Bin 10508 -> 27497 bytes
local/twitter-history-elasticsearch/pom.xml | 46 +++++++++++++++++++
.../twitter/TwitterHistoryElasticsearch.java | 3 +-
.../resources/TwitterHistoryElasticsearch.dot | 6 +--
.../twitter-history-elasticsearch.png | Bin 10508 -> 0 bytes
6 files changed, 65 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8b469d25/local/twitter-history-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/README.md b/local/twitter-history-elasticsearch/README.md
index 649953b..0ccce34 100644
--- a/local/twitter-history-elasticsearch/README.md
+++ b/local/twitter-history-elasticsearch/README.md
@@ -53,27 +53,26 @@ Example Configuration:
type = activity
}
-In the Twitter section you should place all of your relevant authentication keys and whichever Twitter IDs you're looking to follow
-Twitter IDs can be converted from screennames at http://www.gettwitterid.com
+In the Twitter section you should place all of your relevant authentication keys and whichever Twitter IDs you want to pull history for.
-Running:
---------
+Twitter IDs can be converted from screennames at http://www.gettwitterid.com
-You will need to run `./install_templates.sh` in the resources folder in order to apply the templates to your ES cluster
+Build:
+---------
- java -cp target/twitter-history-elasticsearch-0.1-SNAPSHOT.jar -Dconfig.file=application.conf org.apache.streams.twitter.example.TwitterHistoryElasticsearchActivity
+`mvn clean package verify`
-Note that you must modify src/main/resources/application.conf, and supply an absolute path to config.file
+Run:
+--------
-Verification:
--------------
-Open up http://localhost:9200/_plugin/head/ and confirm that the index you specified now contains has data
+ java -cp target/twitter-history-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=application.conf org.apache.streams.example.twitter.TwitterHistoryElasticsearch
-Download https://github.com/w2ogroup/streams-examples/blob/master/twitter-history-elasticsearch/src/main/resources/reports/ActivityReport.json
+Deploy:
+--------
+`mvn -Pdocker clean package docker:build`
-Open up http://localhost:9200/_plugin/marvel and from the folder icon in the top right hand corner click
- Load -> Advanced -> Choose File and select the report you downloaded
+`docker tag twitter-history-elasticsearch:0.2-incubating-SNAPSHOT <dockerregistry>:elasticsearch-reindex:0.2-incubating-SNAPSHOT`
-The gear on the top-right allows you to change the report index
+`docker push <dockerregistry>:twitter-history-elasticsearch:0.2-incubating-SNAPSHOT`
-You should now see dashboards displaying metrics about your twitter activity
\ No newline at end of file
+`docker run <dockerregistry>:twitter-history-elasticsearch:0.2-incubating-SNAPSHOT.jar java -cp twitter-history-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.twitter.TwitterHistoryElasticsearch`
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8b469d25/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png b/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png
index d17e1cd..0a87125 100644
Binary files a/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png and b/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8b469d25/local/twitter-history-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/pom.xml b/local/twitter-history-elasticsearch/pom.xml
index 06855c1..4adabb2 100644
--- a/local/twitter-history-elasticsearch/pom.xml
+++ b/local/twitter-history-elasticsearch/pom.xml
@@ -96,6 +96,12 @@
<groupId>org.apache.streams</groupId>
<artifactId>streams-provider-twitter</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
@@ -293,4 +299,44 @@
</plugin>
</plugins>
</build>
+
+ <profiles>
+ <profile>
+ <id>docker</id>
+ <build>
+ <plugins>
+ <plugin>
+ <!-- The Docker Maven plugin is used to create docker image with the fat jar -->
+ <groupId>org.jolokia</groupId>
+ <artifactId>docker-maven-plugin</artifactId>
+ <version>0.11.0</version>
+ <configuration>
+ <images>
+
+ <image>
+ <alias>${project.artifactId}</alias>
+ <name>${project.artifactId}:${project.version}</name>
+ <build>
+ <from>dockerfile/java:oracle-java8</from>
+ <assembly>
+ <basedir>/</basedir>
+ <descriptorRef>artifact</descriptorRef>
+ </assembly>
+ <!-- Default command for the build image -->
+ </build>
+
+ </image>
+
+ </images>
+ </configuration>
+
+ </plugin>
+
+ </plugins>
+ </build>
+ <activation>
+ <activeByDefault/>
+ </activation>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8b469d25/local/twitter-history-elasticsearch/src/main/java/org/apache/streams/example/twitter/TwitterHistoryElasticsearch.java
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/main/java/org/apache/streams/example/twitter/TwitterHistoryElasticsearch.java b/local/twitter-history-elasticsearch/src/main/java/org/apache/streams/example/twitter/TwitterHistoryElasticsearch.java
index 2e1c886..49dcd42 100644
--- a/local/twitter-history-elasticsearch/src/main/java/org/apache/streams/example/twitter/TwitterHistoryElasticsearch.java
+++ b/local/twitter-history-elasticsearch/src/main/java/org/apache/streams/example/twitter/TwitterHistoryElasticsearch.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.typesafe.config.Config;
import org.apache.streams.config.ComponentConfigurator;
import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.converter.ActivityConverterProcessor;
import org.apache.streams.core.StreamBuilder;
import org.apache.streams.elasticsearch.ElasticsearchConfigurator;
import org.apache.streams.elasticsearch.ElasticsearchPersistWriter;
@@ -76,7 +77,7 @@ public class TwitterHistoryElasticsearch implements Runnable {
public void run() {
TwitterTimelineProvider provider = new TwitterTimelineProvider(config.getTwitter(), ObjectNode.class);
- TwitterTypeConverter converter = new TwitterTypeConverter(ObjectNode.class, Activity.class);
+ ActivityConverterProcessor converter = new ActivityConverterProcessor();
ElasticsearchPersistWriter writer = new ElasticsearchPersistWriter(config.getElasticsearch());
StreamBuilder builder = new LocalStreamBuilder(500);
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8b469d25/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot b/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot
index e13af61..525c61d 100644
--- a/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot
+++ b/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot
@@ -4,7 +4,7 @@ digraph g {
TwitterTimelineProvider [label="TwitterTimelineProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java"];
//processors
- TwitterTypeConverter [label="TwitterTypeConverter",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java"];
+ ActivityConverterProcessor [label="ActivityConverterProcessor",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterProcessor.java"];
//persisters
ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"];
@@ -13,8 +13,8 @@ digraph g {
es [label="es://{index}/{type}",shape=box];
//stream
- TwitterTimelineProvider -> TwitterTypeConverter [label="ObjectNode"];
- TwitterTypeConverter -> ElasticsearchPersistWriter [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+ TwitterTimelineProvider -> ActivityConverterProcessor [label="ObjectNode"];
+ ActivityConverterProcessor -> ElasticsearchPersistWriter [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
ElasticsearchPersistWriter -> es [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/8b469d25/local/twitter-history-elasticsearch/twitter-history-elasticsearch.png
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/twitter-history-elasticsearch.png b/local/twitter-history-elasticsearch/twitter-history-elasticsearch.png
deleted file mode 100644
index d17e1cd..0000000
Binary files a/local/twitter-history-elasticsearch/twitter-history-elasticsearch.png and /dev/null differ