You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2015/05/16 20:44:26 UTC
[02/12] incubator-streams-examples git commit: all files needed for
decent maven site
all files needed for decent maven site
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/107852b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/107852b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/107852b2
Branch: refs/heads/0.2-incubating-rc1
Commit: 107852b258d6a6d31ba9e9447105a1a2f88f0aed
Parents: a1836d0
Author: Steve Blackmon <sb...@apache.org>
Authored: Mon Apr 27 22:25:14 2015 -0500
Committer: Steve Blackmon <sb...@apache.org>
Committed: Mon Apr 27 22:25:14 2015 -0500
----------------------------------------------------------------------
local/elasticsearch-hdfs/ElasticsearchHdfs.md | 47 ----
local/elasticsearch-hdfs/HdfsElasticsearch.md | 50 ----
local/elasticsearch-hdfs/README.md | 4 +-
local/elasticsearch-hdfs/pom.xml | 19 +-
.../main/resources/ElasticsearchHdfs.dot.png | Bin 0 -> 23205 bytes
.../main/resources/HdfsElasticsearch.dot.png | Bin 0 -> 23439 bytes
.../src/site/markdown/ElasticsearchHdfs.md | 47 ++++
.../src/site/markdown/HdfsElasticsearch.md | 50 ++++
.../src/site/markdown/index.md | 35 +++
.../src/site/resources/ElasticsearchHdfs.dot | 17 ++
.../site/resources/ElasticsearchHdfs.dot.png | Bin 0 -> 23205 bytes
.../ElasticsearchHdfsConfiguration.json | 10 +
.../src/site/resources/HdfsElasticsearch.dot | 17 ++
.../site/resources/HdfsElasticsearch.dot.png | Bin 0 -> 23439 bytes
.../HdfsElasticsearchConfiguration.json | 10 +
.../src/site/resources/testBackup.json | 24 ++
.../src/site/resources/testRestore.json | 20 ++
.../ElasticsearchReindex.png | Bin 8913 -> 0 bytes
local/elasticsearch-reindex/README.md | 4 +-
local/elasticsearch-reindex/pom.xml | 18 +-
.../main/resources/ElasticsearchReindex.dot.png | Bin 0 -> 23187 bytes
.../src/site/markdown/index.md | 79 +++++++
.../resources/ActivityChildObjectParent.json | 15 ++
.../src/site/resources/ElasticsearchReindex.dot | 17 ++
.../site/resources/ElasticsearchReindex.dot.png | Bin 0 -> 23187 bytes
.../ElasticsearchReindexConfiguration.json | 10 +
.../src/site/resources/application.json | 26 ++
.../src/site/resources/testReindex.json | 26 ++
.../src/site/resources/testReindexChild.json | 26 ++
.../src/site/resources/testReindexParent.json | 26 ++
.../MongoElasticsearchSync.png | Bin 11502 -> 0 bytes
local/mongo-elasticsearch-sync/README.md | 4 +-
local/mongo-elasticsearch-sync/pom.xml | 3 +
.../resources/MongoElasticsearchSync.dot.png | Bin 0 -> 27856 bytes
.../src/site/markdown/index.md | 68 ++++++
.../site/resources/MongoElasticsearchSync.dot | 18 ++
.../resources/MongoElasticsearchSync.dot.png | Bin 0 -> 27856 bytes
.../MongoElasticsearchSyncConfiguration.json | 10 +
.../src/site/resources/application.json | 17 ++
.../src/site/resources/testSync.json | 18 ++
local/twitter-follow-graph/README.md | 2 +-
.../twitter-follow-graph/TwitterFollowGraph.png | Bin 35826 -> 0 bytes
local/twitter-follow-graph/pom.xml | 21 +-
.../main/resources/TwitterFollowGraph.dot.png | Bin 0 -> 34724 bytes
.../src/site/markdown/index.md | 97 ++++++++
.../src/site/resources/TwitterFollowGraph.dot | 21 ++
.../site/resources/TwitterFollowGraph.dot.png | Bin 0 -> 34724 bytes
.../TwitterFollowGraphConfiguration.json | 13 +
local/twitter-history-elasticsearch/README.md | 2 +-
.../TwitterHistoryElasticsearch.png | Bin 27497 -> 0 bytes
local/twitter-history-elasticsearch/pom.xml | 14 +-
.../TwitterHistoryElasticsearch.dot.png | Bin 0 -> 26774 bytes
.../src/site/markdown/index.md | 78 ++++++
.../resources/TwitterHistoryElasticsearch.dot | 21 ++
.../TwitterHistoryElasticsearch.dot.png | Bin 0 -> 26774 bytes
.../src/site/resources/application.conf | 23 ++
...witterHistoryElasticsearchConfiguration.json | 13 +
.../twitter-userstream-elasticsearch/README.md | 4 +-
.../TwitterUserstreamElasticsearch.png | Bin 65713 -> 0 bytes
local/twitter-userstream-elasticsearch/pom.xml | 12 +-
.../TwitterUserstreamElasticsearch.dot.png | Bin 0 -> 64469 bytes
.../src/site/markdown/index.md | 73 ++++++
.../TwitterUserstreamElasticsearch.dot | 28 +++
.../TwitterUserstreamElasticsearch.dot.png | Bin 0 -> 64469 bytes
.../src/site/resources/application.conf | 22 ++
.../twitter/TwitterUserstreamElasticsearch.json | 14 ++
pom.xml | 235 ++++++++++++++++++-
src/site/site.xml | 29 +++
68 files changed, 1293 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/ElasticsearchHdfs.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/ElasticsearchHdfs.md b/local/elasticsearch-hdfs/ElasticsearchHdfs.md
deleted file mode 100644
index b42d1a8..0000000
--- a/local/elasticsearch-hdfs/ElasticsearchHdfs.md
+++ /dev/null
@@ -1,47 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-elasticsearch-hdfs
-==============================
-
-Description:
------------------
-
-Copies documents from elasticsearch to hdfs.
-
-Specification:
------------------
-
-[ElasticsearchHdfs.dot](src/main/resources/ElasticsearchHdfs.dot "ElasticsearchHdfs.dot" )
-
-Diagram:
------------------
-
-
-
-Example Configuration:
-----------------------
-
- {
- "source": {
- "host": "localhost",
- "port": 27017,
- "db": "streams",
- "collection": "activities"
- },
- "destination": {
- "hosts": [
- "localhost"
- ],
- "port": 9300,
- "clusterName": "elasticsearch",
- "index": "destination",
- "type": "activity"
- }
- }
-
-Run:
---------
-
-`docker run elasticsearch-hdfs:0.2-incubating-SNAPSHOT-PP.jar java -cp stash-migrate-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.elasticsearch.example.HdfsElasticsearch`
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/HdfsElasticsearch.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/HdfsElasticsearch.md b/local/elasticsearch-hdfs/HdfsElasticsearch.md
deleted file mode 100644
index c4ca261..0000000
--- a/local/elasticsearch-hdfs/HdfsElasticsearch.md
+++ /dev/null
@@ -1,50 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-hdfs-elasticsearch
-==============================
-
-Description:
------------------
-
-Copies documents from hdfs to elasticsearch.
-
-Specification:
------------------
-
-[HdfsElasticsearch.dot](src/main/resources/HdfsElasticsearch.dot "HdfsElasticsearch.dot" )
-
-Diagram:
------------------
-
-
-
-Example Configuration:
-----------------------
-
- {
- "source": {
- "scheme": "file",
- "host": "localhost",
- "user": "cloudera",
- "path": "/tmp",
- "writerPath": "activity"
- },
- "destination": {
- "hosts": [
- "localhost"
- ],
- "port": 9300,
- "clusterName": "elasticsearch",
- "index": "activity2",
- "type": "activity",
- "forceUseConfig": true
- }
- }
-
-
-Run:
---------
-
-`docker run elasticsearch-hdfs:0.2-incubating-SNAPSHOT.jar java -cp elasticsearch-hdfs-0.2-incubating-SNAPSHOT.jar -Dconfig.file=file://<location_of_config_file>.json org.apache.streams.elasticsearch.example.HdfsElasticsearch`
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/README.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/README.md b/local/elasticsearch-hdfs/README.md
index 349046d..b3e2e3f 100644
--- a/local/elasticsearch-hdfs/README.md
+++ b/local/elasticsearch-hdfs/README.md
@@ -16,9 +16,9 @@ Copies documents between elasticsearch and file system using the hdfs persist mo
Streams:
--------
-[ElasticsearchHdfs](ElasticsearchHdfs.md "ElasticsearchHdfs" )
+[ElasticsearchHdfs](src/site/markdown/ElasticsearchHdfs.md "ElasticsearchHdfs" )
-[HdfsElasticsearch](HdfsElasticsearch.md "HdfsElasticsearch" )
+[HdfsElasticsearch](src/site/markdown/HdfsElasticsearch.md "HdfsElasticsearch" )
Build:
---------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/pom.xml b/local/elasticsearch-hdfs/pom.xml
index 8819319..3d07371 100644
--- a/local/elasticsearch-hdfs/pom.xml
+++ b/local/elasticsearch-hdfs/pom.xml
@@ -29,7 +29,9 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>elasticsearch-hdfs</artifactId>
- <version>0.2-incubating-SNAPSHOT</version>
+ <name>elasticsearch-hdfs</name>
+
+ <description>Copies documents between elasticsearch and file system using the hdfs persist module.</description>
<properties>
<elasticsearch.version>1.1.0</elasticsearch.version>
@@ -71,45 +73,38 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
<artifactId>streams-config</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-util</artifactId>
- <version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-runtime-local</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-persist-elasticsearch</artifactId>
- <version>0.2-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-persist-hdfs</artifactId>
- <version>0.2-incubating-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.dot.png
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.dot.png b/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.dot.png
new file mode 100644
index 0000000..fd5f3b5
Binary files /dev/null and b/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.dot.png
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.dot.png b/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.dot.png
new file mode 100644
index 0000000..d44f8db
Binary files /dev/null and b/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/markdown/ElasticsearchHdfs.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/markdown/ElasticsearchHdfs.md b/local/elasticsearch-hdfs/src/site/markdown/ElasticsearchHdfs.md
new file mode 100644
index 0000000..219516a
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/site/markdown/ElasticsearchHdfs.md
@@ -0,0 +1,47 @@
+Apache Streams (incubating)
+Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------
+
+elasticsearch-hdfs
+==============================
+
+Description:
+-----------------
+
+Copies documents from elasticsearch to hdfs.
+
+Specification:
+-----------------
+
+[ElasticsearchHdfs.dot](ElasticsearchHdfs.dot "ElasticsearchHdfs.dot" )
+
+Diagram:
+-----------------
+
+
+
+Example Configuration:
+----------------------
+
+ {
+ "source": {
+ "host": "localhost",
+ "port": 27017,
+ "db": "streams",
+ "collection": "activities"
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "destination",
+ "type": "activity"
+ }
+ }
+
+Run:
+--------
+
+`docker run elasticsearch-hdfs:0.2-incubating-SNAPSHOT-PP.jar java -cp stash-migrate-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.elasticsearch.example.HdfsElasticsearch`
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/markdown/HdfsElasticsearch.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/markdown/HdfsElasticsearch.md b/local/elasticsearch-hdfs/src/site/markdown/HdfsElasticsearch.md
new file mode 100644
index 0000000..db6b862
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/site/markdown/HdfsElasticsearch.md
@@ -0,0 +1,50 @@
+Apache Streams (incubating)
+Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------
+
+hdfs-elasticsearch
+==============================
+
+Description:
+-----------------
+
+Copies documents from hdfs to elasticsearch.
+
+Specification:
+-----------------
+
+[HdfsElasticsearch.dot](HdfsElasticsearch.dot "HdfsElasticsearch.dot" )
+
+Diagram:
+-----------------
+
+
+
+Example Configuration:
+----------------------
+
+ {
+ "source": {
+ "scheme": "file",
+ "host": "localhost",
+ "user": "cloudera",
+ "path": "/tmp",
+ "writerPath": "activity"
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "activity2",
+ "type": "activity",
+ "forceUseConfig": true
+ }
+ }
+
+
+Run:
+--------
+
+`docker run elasticsearch-hdfs:0.2-incubating-SNAPSHOT.jar java -cp elasticsearch-hdfs-0.2-incubating-SNAPSHOT.jar -Dconfig.file=file://<location_of_config_file>.json org.apache.streams.elasticsearch.example.HdfsElasticsearch`
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/markdown/index.md b/local/elasticsearch-hdfs/src/site/markdown/index.md
new file mode 100644
index 0000000..cfd5a30
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/site/markdown/index.md
@@ -0,0 +1,35 @@
+Apache Streams (incubating)
+Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------
+
+elasticsearch-hdfs
+==============================
+
+Requirements:
+-------------
+ - A running ElasticSearch 1.0.0+ instance
+
+Description:
+------------
+Copies documents between elasticsearch and file system using the hdfs persist module.
+
+Streams:
+--------
+
+<a href="HdfsElasticsearch.html" target="_self">HdfsElasticsearch</a>
+
+<a href="ElasticsearchHdfs.html" target="_self">ElasticsearchHdfs</a>
+
+Build:
+---------
+
+`mvn clean package`
+
+Note that an alternative version of hdfs is packaged, by excluding org.apache.hadoop.hadoop-hdfs when
+importing org.apache.streams.streams-persist-hdfs, and specifically depending on a different preferred version.
+
+Deploy:
+--------
+
+`mvn -Pdocker clean package docker:build`
+
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot
new file mode 100644
index 0000000..c953522
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot
@@ -0,0 +1,17 @@
+digraph g {
+
+ //providers
+ ElasticsearchPersistReader [label="ElasticsearchPersistReader",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java"];
+
+ //persisters
+ WebHdfsPersistWriter [label="WebHdfsPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"];
+
+ //data
+ source [label="es://{indexes}/{types}",shape=box];
+ destination [label="hdfs://{index}/{type}",shape=box];
+
+ //stream
+ source -> ElasticsearchPersistReader
+ ElasticsearchPersistReader -> WebHdfsPersistWriter [label="String"];
+ WebHdfsPersistWriter -> destination
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot.png
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot.png b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot.png
new file mode 100644
index 0000000..fd5f3b5
Binary files /dev/null and b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfsConfiguration.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfsConfiguration.json b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfsConfiguration.json
new file mode 100644
index 0000000..91324dd
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfsConfiguration.json
@@ -0,0 +1,10 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "type": "object",
+ "javaType" : "org.apache.streams.elasticsearch.example.ElasticsearchHdfsConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "source": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchReaderConfiguration", "type": "object", "required": true },
+ "destination": { "javaType": "org.apache.streams.hdfs.HdfsWriterConfiguration", "type": "object", "required": true }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot
new file mode 100644
index 0000000..f9a0efa
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot
@@ -0,0 +1,17 @@
+digraph g {
+
+ //providers
+ WebHdfsPersistReader [label="WebHdfsPersistReader",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java"];
+
+ //persisters
+ ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"];
+
+ //data
+ source [label="hdfs://{indexes}/{types}",shape=box];
+ destination [label="es://{index}/{type}",shape=box];
+
+ //stream
+ source -> WebHdfsPersistReader
+ WebHdfsPersistReader -> ElasticsearchPersistWriter [label="String"];
+ ElasticsearchPersistWriter -> destination
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot.png
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot.png b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot.png
new file mode 100644
index 0000000..d44f8db
Binary files /dev/null and b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearchConfiguration.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearchConfiguration.json b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearchConfiguration.json
new file mode 100644
index 0000000..c8072ba
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearchConfiguration.json
@@ -0,0 +1,10 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "type": "object",
+ "javaType" : "org.apache.streams.elasticsearch.example.HdfsElasticsearchConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "source": { "javaType": "org.apache.streams.hdfs.HdfsReaderConfiguration", "type": "object", "required": true },
+ "destination": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration", "type": "object", "required": true }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/testBackup.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/resources/testBackup.json b/local/elasticsearch-hdfs/src/site/resources/testBackup.json
new file mode 100644
index 0000000..2002cb8
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/site/resources/testBackup.json
@@ -0,0 +1,24 @@
+{
+ "source": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "indexes": [
+ "source"
+ ],
+ "types": [
+ "activity"
+ ]
+ },
+ "destination": {
+ "scheme": "file",
+ "host": "localhost",
+ "user": "cloudera",
+ "path": "target",
+ "writerPath": "test",
+ "writerFilePrefix": "activities"
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/testRestore.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/resources/testRestore.json b/local/elasticsearch-hdfs/src/site/resources/testRestore.json
new file mode 100644
index 0000000..63b42c5
--- /dev/null
+++ b/local/elasticsearch-hdfs/src/site/resources/testRestore.json
@@ -0,0 +1,20 @@
+{
+ "source": {
+ "scheme": "file",
+ "host": "localhost",
+ "user": "cloudera",
+ "path": "target",
+ "readerPath": "test"
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "destination",
+ "type": "activity",
+ "forceUseConfig": true
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/ElasticsearchReindex.png
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/ElasticsearchReindex.png b/local/elasticsearch-reindex/ElasticsearchReindex.png
deleted file mode 100644
index 9e59093..0000000
Binary files a/local/elasticsearch-reindex/ElasticsearchReindex.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/README.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/README.md b/local/elasticsearch-reindex/README.md
index 5fbace2..71e8c28 100644
--- a/local/elasticsearch-reindex/README.md
+++ b/local/elasticsearch-reindex/README.md
@@ -13,7 +13,7 @@ Requirements:
Description:
------------
-Copies documents into a different index
+Copies documents between indexes.
Specification:
-----------------
@@ -23,7 +23,7 @@ Specification:
Diagram:
-----------------
-
+
Example Configuration:
----------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/pom.xml
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/pom.xml b/local/elasticsearch-reindex/pom.xml
index a75fa2a..59e968b 100644
--- a/local/elasticsearch-reindex/pom.xml
+++ b/local/elasticsearch-reindex/pom.xml
@@ -29,7 +29,9 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>elasticsearch-reindex</artifactId>
- <version>0.2-incubating-SNAPSHOT</version>
+ <name>elasticsearch-reindex</name>
+
+ <description>Copies documents between indexes.</description>
<properties>
<elasticsearch.version>1.1.0</elasticsearch.version>
@@ -63,40 +65,34 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
<artifactId>streams-config</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-util</artifactId>
- <version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-runtime-local</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-persist-elasticsearch</artifactId>
- <version>0.2-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/main/resources/ElasticsearchReindex.dot.png
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/main/resources/ElasticsearchReindex.dot.png b/local/elasticsearch-reindex/src/main/resources/ElasticsearchReindex.dot.png
new file mode 100644
index 0000000..b6541bb
Binary files /dev/null and b/local/elasticsearch-reindex/src/main/resources/ElasticsearchReindex.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/markdown/index.md b/local/elasticsearch-reindex/src/site/markdown/index.md
new file mode 100644
index 0000000..df20698
--- /dev/null
+++ b/local/elasticsearch-reindex/src/site/markdown/index.md
@@ -0,0 +1,79 @@
+Apache Streams (incubating)
+Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------
+
+elasticsearch-reindex
+==============================
+
+Requirements:
+-------------
+ - A running ElasticSearch 1.0.0+ cluster
+ - Transport client access to cluster
+ - elasticsearch.version and lucene.version set to match cluster
+
+Description:
+------------
+Copies documents into a different index
+
+Specification:
+-----------------
+
+[ElasticsearchReindex.dot](src/main/resources/ElasticsearchReindex.dot "ElasticsearchReindex.dot" )
+
+Diagram:
+-----------------
+
+
+
+Example Configuration:
+----------------------
+
+ {
+ "source": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "indexes": [
+ "activity"
+ ],
+ "types": [
+ "activity"
+ ],
+ "forceUseConfig": true
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "activity2",
+ "type": "activity",
+ "forceUseConfig": true
+ }
+ }
+
+Populate source and destination in configuration with cluster / index / type details.
+
+Build:
+---------
+
+`mvn clean package verify`
+
+Run:
+--------
+
+`java -cp target/elasticsearch-reindex-0.1-SNAPSHOT.jar -Dconfig.file=src/main/resources/application.json org.apache.streams.example.elasticsearch.ElasticsearchReindex`
+
+Deploy:
+--------
+`mvn -Pdocker clean package docker:build`
+
+`docker tag elasticsearch-reindex:0.2-incubating-SNAPSHOT <dockerregistry>:elasticsearch-reindex:0.2-incubating-SNAPSHOT`
+
+`docker push <dockerregistry>:elasticsearch-reindex:0.2-incubating-SNAPSHOT`
+
+`docker run <dockerregistry>:elasticsearch-reindex:0.2-incubating-SNAPSHOT.jar java -cp elasticsearch-reindex-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.elasticsearch.ElasticsearchReindex`
+
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/ActivityChildObjectParent.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/resources/ActivityChildObjectParent.json b/local/elasticsearch-reindex/src/site/resources/ActivityChildObjectParent.json
new file mode 100644
index 0000000..963fbe7
--- /dev/null
+++ b/local/elasticsearch-reindex/src/site/resources/ActivityChildObjectParent.json
@@ -0,0 +1,15 @@
+{
+ "template": "*",
+ "order": 100,
+ "mappings": {
+ "object": {
+ "dynamic": true
+ },
+ "activity": {
+ "_parent": {
+ "type": "object"
+ },
+ "dynamic": true
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot
new file mode 100644
index 0000000..52456a9
--- /dev/null
+++ b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot
@@ -0,0 +1,17 @@
+digraph g {
+
+ //providers
+ ElasticsearchPersistReader [label="ElasticsearchPersistReader",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java"];
+
+ //persisters
+ ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"];
+
+ //data
+ source [label="es://{indexes}/{types}",shape=box];
+ destination [label="es://{index}/{type}",shape=box];
+
+ //stream
+ source -> ElasticsearchPersistReader
+ ElasticsearchPersistReader -> ElasticsearchPersistWriter [label="String"];
+ ElasticsearchPersistWriter -> destination
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot.png
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot.png b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot.png
new file mode 100644
index 0000000..b6541bb
Binary files /dev/null and b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindexConfiguration.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindexConfiguration.json b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindexConfiguration.json
new file mode 100644
index 0000000..ffdc4d8
--- /dev/null
+++ b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindexConfiguration.json
@@ -0,0 +1,10 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "type": "object",
+ "javaType" : "org.apache.streams.elasticsearch.example.ElasticsearchReindexConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "source": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchReaderConfiguration", "type": "object", "required": true },
+ "destination": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration", "type": "object", "required": true }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/application.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/resources/application.json b/local/elasticsearch-reindex/src/site/resources/application.json
new file mode 100644
index 0000000..567b19d
--- /dev/null
+++ b/local/elasticsearch-reindex/src/site/resources/application.json
@@ -0,0 +1,26 @@
+{
+ "source": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "indexes": [
+ "activity"
+ ],
+ "types": [
+ "activity"
+ ],
+ "forceUseConfig": true
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "activity2",
+ "type": "activity",
+ "forceUseConfig": true
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/testReindex.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/resources/testReindex.json b/local/elasticsearch-reindex/src/site/resources/testReindex.json
new file mode 100644
index 0000000..0cc5aa0
--- /dev/null
+++ b/local/elasticsearch-reindex/src/site/resources/testReindex.json
@@ -0,0 +1,26 @@
+{
+ "source": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "indexes": [
+ "source"
+ ],
+ "types": [
+ "activity"
+ ]
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "destination",
+ "type": "activity",
+ "forceUseConfig": true
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/testReindexChild.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/resources/testReindexChild.json b/local/elasticsearch-reindex/src/site/resources/testReindexChild.json
new file mode 100644
index 0000000..0cc5aa0
--- /dev/null
+++ b/local/elasticsearch-reindex/src/site/resources/testReindexChild.json
@@ -0,0 +1,26 @@
+{
+ "source": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "indexes": [
+ "source"
+ ],
+ "types": [
+ "activity"
+ ]
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "destination",
+ "type": "activity",
+ "forceUseConfig": true
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/testReindexParent.json
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/resources/testReindexParent.json b/local/elasticsearch-reindex/src/site/resources/testReindexParent.json
new file mode 100644
index 0000000..c601cea
--- /dev/null
+++ b/local/elasticsearch-reindex/src/site/resources/testReindexParent.json
@@ -0,0 +1,26 @@
+{
+ "source": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "indexes": [
+ "source"
+ ],
+ "types": [
+ "object"
+ ]
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "destination",
+ "type": "object",
+ "forceUseConfig": true
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/MongoElasticsearchSync.png
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/MongoElasticsearchSync.png b/local/mongo-elasticsearch-sync/MongoElasticsearchSync.png
deleted file mode 100644
index a8fc4d7..0000000
Binary files a/local/mongo-elasticsearch-sync/MongoElasticsearchSync.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/README.md
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/README.md b/local/mongo-elasticsearch-sync/README.md
index 72b11db..ddfb84b 100644
--- a/local/mongo-elasticsearch-sync/README.md
+++ b/local/mongo-elasticsearch-sync/README.md
@@ -12,7 +12,7 @@ Requirements:
Description:
------------
-Copies documents from mongodb to elasticsearch
+Copies documents from mongodb to elasticsearch.
Specification:
-----------------
@@ -22,7 +22,7 @@ Specification:
Diagram:
-----------------
-
+
Example Configuration:
----------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/pom.xml
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/pom.xml b/local/mongo-elasticsearch-sync/pom.xml
index 4d271e0..411bd9b 100644
--- a/local/mongo-elasticsearch-sync/pom.xml
+++ b/local/mongo-elasticsearch-sync/pom.xml
@@ -29,6 +29,9 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>mongo-elasticsearch-sync</artifactId>
+ <name>mongo-elasticsearch-sync</name>
+
+ <description>Copies documents from mongodb to elasticsearch.</description>
<properties>
<elasticsearch.version>1.1.0</elasticsearch.version>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/main/resources/MongoElasticsearchSync.dot.png
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/src/main/resources/MongoElasticsearchSync.dot.png b/local/mongo-elasticsearch-sync/src/main/resources/MongoElasticsearchSync.dot.png
new file mode 100644
index 0000000..b898bfc
Binary files /dev/null and b/local/mongo-elasticsearch-sync/src/main/resources/MongoElasticsearchSync.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/src/site/markdown/index.md b/local/mongo-elasticsearch-sync/src/site/markdown/index.md
new file mode 100644
index 0000000..c4f0b9e
--- /dev/null
+++ b/local/mongo-elasticsearch-sync/src/site/markdown/index.md
@@ -0,0 +1,68 @@
+Apache Streams (incubating)
+Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------
+
+mongo-elasticsearch-sync
+==============================
+
+Requirements:
+-------------
+ - A running MongoDB 2.4+ instance
+ - A running ElasticSearch 1.0.0+ instance
+
+Description:
+------------
+Copies documents from mongodb to elasticsearch
+
+Specification:
+-----------------
+
+[MongoElasticsearchSync.dot](MongoElasticsearchSync.dot "MongoElasticsearchSync.dot" )
+
+Diagram:
+-----------------
+
+
+
+Example Configuration:
+----------------------
+
+ {
+ "source": {
+ "host": "localhost",
+ "port": 27017,
+ "db": "streams",
+ "collection": "activities"
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "destination",
+ "type": "activity"
+ }
+ }
+
+Build:
+---------
+
+`mvn clean package verify`
+
+Run:
+--------
+
+`java -cp target/mongo-elasticsearch-sync-0.1-SNAPSHOT.jar -Dconfig.file=src/main/resources/application.json org.apache.streams.example.elasticsearch.MongoElasticsearchSync`
+
+Deploy:
+--------
+
+`mvn -Pdocker clean package docker:build`
+
+`docker tag mongo-elasticsearch-sync:0.2-incubating-SNAPSHOT <dockerregistry>:mongo-elasticsearch-sync:0.2-incubating-SNAPSHOT`
+
+`docker push <dockerregistry>:mongo-elasticsearch-sync:0.2-incubating-SNAPSHOT`
+
+`docker run <dockerregistry>:mongo-elasticsearch-sync:0.2-incubating-SNAPSHOT java -cp mongo-elasticsearch-sync-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.elasticsearch.MongoElasticsearchSync`
+
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot
new file mode 100644
index 0000000..e3babf5
--- /dev/null
+++ b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot
@@ -0,0 +1,18 @@
+digraph g {
+
+ //providers
+ MongoPersistReader [label="MongoPersistReader",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistReader.java"];
+
+ //persisters
+ ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"];
+
+ //data
+ source [label="mongdb://{db}/{collection}",shape=box];
+ destination [label="es://{index}/{type}",shape=box];
+
+ //stream
+ source -> MongoPersistReader [label="ObjectNode"];
+ MongoPersistReader -> ElasticsearchPersistWriter [label="ObjectNode"];
+ ElasticsearchPersistWriter -> destination [label="ObjectNode"];
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot.png
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot.png b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot.png
new file mode 100644
index 0000000..b898bfc
Binary files /dev/null and b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSyncConfiguration.json
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSyncConfiguration.json b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSyncConfiguration.json
new file mode 100644
index 0000000..a592699
--- /dev/null
+++ b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSyncConfiguration.json
@@ -0,0 +1,10 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "type": "object",
+ "javaType" : "org.apache.streams.example.elasticsearch.MongoElasticsearchSyncConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "source": { "javaType": "org.apache.streams.mongo.MongoConfiguration", "type": "object", "required": true },
+ "destination": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration", "type": "object", "required": true }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/application.json
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/src/site/resources/application.json b/local/mongo-elasticsearch-sync/src/site/resources/application.json
new file mode 100644
index 0000000..facb336
--- /dev/null
+++ b/local/mongo-elasticsearch-sync/src/site/resources/application.json
@@ -0,0 +1,17 @@
+{
+ "source": {
+ "host": "localhost",
+ "port": 27017,
+ "db": "streams",
+ "collection": "activities"
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "destination",
+ "type": "activity"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/testSync.json
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/src/site/resources/testSync.json b/local/mongo-elasticsearch-sync/src/site/resources/testSync.json
new file mode 100644
index 0000000..b678c21
--- /dev/null
+++ b/local/mongo-elasticsearch-sync/src/site/resources/testSync.json
@@ -0,0 +1,18 @@
+{
+ "source": {
+ "host": "localhost",
+ "port": 37017,
+ "db": "local",
+ "collection": "activities"
+ },
+ "destination": {
+ "hosts": [
+ "localhost"
+ ],
+ "port": 9300,
+ "clusterName": "elasticsearch",
+ "index": "destination",
+ "type": "activity",
+ "forceUseConfig": true
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/README.md
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/README.md b/local/twitter-follow-graph/README.md
index 5921df8..e7e3bce 100644
--- a/local/twitter-follow-graph/README.md
+++ b/local/twitter-follow-graph/README.md
@@ -22,7 +22,7 @@ Specification:
Diagram:
-----------------
-
+
Example Configuration:
----------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/TwitterFollowGraph.png
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/TwitterFollowGraph.png b/local/twitter-follow-graph/TwitterFollowGraph.png
deleted file mode 100644
index b84db6c..0000000
Binary files a/local/twitter-follow-graph/TwitterFollowGraph.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/pom.xml
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/pom.xml b/local/twitter-follow-graph/pom.xml
index 5324462..db827e2 100644
--- a/local/twitter-follow-graph/pom.xml
+++ b/local/twitter-follow-graph/pom.xml
@@ -23,37 +23,38 @@
<parent>
<groupId>org.apache.streams</groupId>
<artifactId>streams-examples-local</artifactId>
- <version>0.2-incubating-SNAPSHOT</version>
+ <version>0.3-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>twitter-follow-graph</artifactId>
+ <name>twitter-follow-graph</name>
+
+ <description>
+ Collects friend or follower connections for a set of twitter users to build a graph database in neo4j.
+ </description>
<dependencies>
<dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
<artifactId>streams-config</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-runtime-local</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-provider-twitter</artifactId>
- <version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
@@ -64,12 +65,10 @@
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-persist-graph</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/main/resources/TwitterFollowGraph.dot.png
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/main/resources/TwitterFollowGraph.dot.png b/local/twitter-follow-graph/src/main/resources/TwitterFollowGraph.dot.png
new file mode 100644
index 0000000..10dc842
Binary files /dev/null and b/local/twitter-follow-graph/src/main/resources/TwitterFollowGraph.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/site/markdown/index.md b/local/twitter-follow-graph/src/site/markdown/index.md
new file mode 100644
index 0000000..a813f19
--- /dev/null
+++ b/local/twitter-follow-graph/src/site/markdown/index.md
@@ -0,0 +1,97 @@
+Apache Streams (incubating)
+Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------
+
+twitter-follow-graph
+==============================
+
+Requirements:
+-------------
+ - Authorized Twitter API credentials
+ - A running Neo4J 1.9.0+ instance
+
+Description:
+------------
+Collects friend or follower connections for a set of twitter users to build a graph database in neo4j.
+
+Specification:
+-----------------
+
+[TwitterFollowGraph.dot](TwitterFollowGraph.dot "TwitterFollowGraph.dot" )
+
+Diagram:
+-----------------
+
+
+
+Example Configuration:
+----------------------
+
+ {
+ "twitter": {
+ "endpoint": "friends",
+ "oauth": {
+ "consumerSecret": "",
+ "consumerKey": "",
+ "accessToken": "",
+ "accessTokenSecret": ""
+ },
+ "info": [
+ 42232950
+ ]
+ },
+ "graph": {
+ "vertices": {
+ "objects": [
+ "actor",
+ "object"
+ ],
+ "verbs": [
+ "follow"
+ ],
+ "objectTypes": [
+ "page"
+ ]
+ },
+ "edges": {
+ "objects": [
+ "actor",
+ "object"
+ ],
+ "verbs": [
+ "follow"
+ ],
+ "objectTypes": [
+ "page"
+ ]
+ },
+ "protocol": "http",
+ "hostname": "localhost",
+ "port": 7474,
+ "requestMethod": "GET",
+ "content-type": "application/json",
+ "type": "neo4j",
+ "graph": "data"
+ }
+ }
+
+Build:
+---------
+
+`mvn clean package verify`
+
+Run:
+--------
+
+`java -cp target/twitter-follow-graph-0.2-incubating-SNAPSHOT.jar -Dconfig.file=src/main/resources/application.json org.apache.streams.example.graph.TwitterFollowGraph`
+
+Deploy:
+--------
+`mvn -Pdocker clean package docker:build`
+
+`docker tag twitter-follow-graph:0.2-incubating-SNAPSHOT <dockerregistry>:twitter-follow-graph:0.2-incubating-SNAPSHOT`
+
+`docker push <dockerregistry>:twitter-follow-graph:0.2-incubating-SNAPSHOT`
+
+`docker run <dockerregistry>:twitter-follow-graph:0.2-incubating-SNAPSHOT java -cp twitter-follow-graph-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.graph.TwitterFollowGraph`
+
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot
new file mode 100644
index 0000000..c14a2ab
--- /dev/null
+++ b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot
@@ -0,0 +1,21 @@
+digraph g {
+
+ //providers
+ TwitterFollowingProvider [label="TwitterFollowingProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProvider.java"];
+
+ //processors
+ TypeConverterProcessor [label="TypeConverterProcessor",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-converters/src/main/java/org/apache/streams/converters/TypeConverterProcessor.java"];
+ ActivityConverterProcessor [label="ActivityConverterProcessor",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-converters/src/main/java/org/apache/streams/converters/ActivityConverterProcessor.java"];
+
+ //persisters
+ GraphPersistWriter [label="GraphPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/GraphPersistWriter.java"];
+
+ //data
+ destination [label="http://{host}:{port}/db/data",shape=box];
+
+ //stream
+ TwitterFollowingProvider -> TypeConverterProcessor [label="Follow",URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.java"];
+ TypeConverterProcessor -> ActivityConverterProcessor [label="String"];
+ ActivityConverterProcessor -> GraphPersistWriter [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+ GraphPersistWriter -> destination
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot.png
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot.png b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot.png
new file mode 100644
index 0000000..10dc842
Binary files /dev/null and b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/site/resources/TwitterFollowGraphConfiguration.json
----------------------------------------------------------------------
diff --git a/local/twitter-follow-graph/src/site/resources/TwitterFollowGraphConfiguration.json b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraphConfiguration.json
new file mode 100644
index 0000000..c2ccc80
--- /dev/null
+++ b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraphConfiguration.json
@@ -0,0 +1,13 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "type": "object",
+ "javaType" : "org.apache.streams.example.graph.TwitterFollowingGraphConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "twitter": { "javaType": "org.apache.streams.twitter.TwitterUserInformationConfiguration", "type": "object", "required": true },
+ "graph": { "javaType": "org.apache.streams.graph.GraphWriterConfiguration", "type": "object", "required": true }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/README.md b/local/twitter-history-elasticsearch/README.md
index 0ccce34..7e64a84 100644
--- a/local/twitter-history-elasticsearch/README.md
+++ b/local/twitter-history-elasticsearch/README.md
@@ -24,7 +24,7 @@ Specification:
Diagram:
-----------------
-
+
Example Configuration:
----------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png b/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png
deleted file mode 100644
index 0a87125..0000000
Binary files a/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/pom.xml b/local/twitter-history-elasticsearch/pom.xml
index 4adabb2..19b9e1c 100644
--- a/local/twitter-history-elasticsearch/pom.xml
+++ b/local/twitter-history-elasticsearch/pom.xml
@@ -28,6 +28,13 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>twitter-history-elasticsearch</artifactId>
+ <name>twitter-history-elasticsearch</name>
+
+ <description>
+ Retrieves as many posts from a known list of users as twitter API allows.
+
+ Converts them to activities, and writes them in activity format to Elasticsearch.
+ </description>
<properties>
<elasticsearch.version>1.1.0</elasticsearch.version>
@@ -68,34 +75,28 @@
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
- <version>1.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-config</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-util</artifactId>
- <version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-runtime-local</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-provider-twitter</artifactId>
- <version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
@@ -106,7 +107,6 @@
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-persist-elasticsearch</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot.png
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot.png b/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot.png
new file mode 100644
index 0000000..9be8aaf
Binary files /dev/null and b/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/site/markdown/index.md b/local/twitter-history-elasticsearch/src/site/markdown/index.md
new file mode 100644
index 0000000..4afcf4e
--- /dev/null
+++ b/local/twitter-history-elasticsearch/src/site/markdown/index.md
@@ -0,0 +1,78 @@
+Apache Streams (incubating)
+Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------
+
+twitter-history-elasticsearch
+==============================
+
+Requirements:
+-------------
+ - Authorized Twitter API credentials
+ - A running ElasticSearch 1.0.0+ instance
+
+Description:
+------------
+Retrieves as many posts from a known list of users as twitter API allows.
+
+Converts them to activities, and writes them in activity format to Elasticsearch.
+
+Specification:
+-----------------
+
+[TwitterHistoryElasticsearch.dot](src/main/resources/TwitterHistoryElasticsearch.dot "TwitterHistoryElasticsearch.dot" )
+
+Diagram:
+-----------------
+
+
+
+Example Configuration:
+----------------------
+
+ twitter {
+ host = "api.twitter.com"
+ endpoint = "statuses/user_timeline"
+ oauth {
+ consumerKey = ""
+ consumerSecret = ""
+ accessToken = ""
+ accessTokenSecret = ""
+ }
+ info = [
+ "42232950"
+ "211620426"
+ ]
+ }
+ elasticsearch {
+ hosts = [
+ localhost
+ ]
+ port = 9300
+ clusterName = elasticsearch
+ index = userhistory_activity
+ type = activity
+ }
+
+In the Twitter section you should place all of your relevant authentication keys and whichever Twitter IDs you want to pull history for.
+
+Twitter IDs can be converted from screennames at http://www.gettwitterid.com
+
+Build:
+---------
+
+`mvn clean package verify`
+
+Run:
+--------
+
+ java -cp target/twitter-history-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=application.conf org.apache.streams.example.twitter.TwitterHistoryElasticsearch
+
+Deploy:
+--------
+`mvn -Pdocker clean package docker:build`
+
+`docker tag twitter-history-elasticsearch:0.2-incubating-SNAPSHOT <dockerregistry>:elasticsearch-reindex:0.2-incubating-SNAPSHOT`
+
+`docker push <dockerregistry>:twitter-history-elasticsearch:0.2-incubating-SNAPSHOT`
+
+`docker run <dockerregistry>:twitter-history-elasticsearch:0.2-incubating-SNAPSHOT.jar java -cp twitter-history-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.twitter.TwitterHistoryElasticsearch`
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot b/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot
new file mode 100644
index 0000000..525c61d
--- /dev/null
+++ b/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot
@@ -0,0 +1,21 @@
+digraph g {
+
+ //providers
+ TwitterTimelineProvider [label="TwitterTimelineProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java"];
+
+ //processors
+ ActivityConverterProcessor [label="ActivityConverterProcessor",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterProcessor.java"];
+
+ //persisters
+ ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"];
+
+ //data
+ es [label="es://{index}/{type}",shape=box];
+
+ //stream
+ TwitterTimelineProvider -> ActivityConverterProcessor [label="ObjectNode"];
+ ActivityConverterProcessor -> ElasticsearchPersistWriter [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+
+ ElasticsearchPersistWriter -> es [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot.png
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot.png b/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot.png
new file mode 100644
index 0000000..9be8aaf
Binary files /dev/null and b/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/resources/application.conf
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/site/resources/application.conf b/local/twitter-history-elasticsearch/src/site/resources/application.conf
new file mode 100644
index 0000000..c74b54b
--- /dev/null
+++ b/local/twitter-history-elasticsearch/src/site/resources/application.conf
@@ -0,0 +1,23 @@
+twitter {
+ host = "api.twitter.com"
+ endpoint = "statuses/user_timeline"
+ oauth {
+ consumerKey = ""
+ consumerSecret = ""
+ accessToken = ""
+ accessTokenSecret = ""
+ }
+ info = [
+ "42232950"
+ "211620426"
+ ]
+}
+elasticsearch {
+ hosts = [
+ localhost
+ ]
+ port = 9300
+ clusterName = elasticsearch
+ index = twitterhistory_activity
+ type = activity
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/resources/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/site/resources/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json b/local/twitter-history-elasticsearch/src/site/resources/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json
new file mode 100644
index 0000000..ea9b165
--- /dev/null
+++ b/local/twitter-history-elasticsearch/src/site/resources/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json
@@ -0,0 +1,13 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "type": "object",
+ "javaType" : "org.apache.streams.example.twitter.TwitterHistoryElasticsearchConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "twitter": { "javaType": "org.apache.streams.twitter.TwitterUserInformationConfiguration", "type": "object", "required": true },
+ "elasticsearch": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration", "type": "object", "required": true }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/README.md b/local/twitter-userstream-elasticsearch/README.md
index e2b4a2d..1b68afa 100644
--- a/local/twitter-userstream-elasticsearch/README.md
+++ b/local/twitter-userstream-elasticsearch/README.md
@@ -12,7 +12,7 @@ Requirements:
Description:
------------
-This example connects to an active twitter account and stores the userstream as activities in Elasticsearch
+Connects to an active twitter account and stores the userstream as activities in Elasticsearch
Specification:
-----------------
@@ -22,7 +22,7 @@ Specification:
Diagram:
-----------------
-
+
Example Configuration:
----------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/TwitterUserstreamElasticsearch.png
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/TwitterUserstreamElasticsearch.png b/local/twitter-userstream-elasticsearch/TwitterUserstreamElasticsearch.png
deleted file mode 100644
index 003a002..0000000
Binary files a/local/twitter-userstream-elasticsearch/TwitterUserstreamElasticsearch.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/pom.xml b/local/twitter-userstream-elasticsearch/pom.xml
index f438e77..6bb57b6 100644
--- a/local/twitter-userstream-elasticsearch/pom.xml
+++ b/local/twitter-userstream-elasticsearch/pom.xml
@@ -28,6 +28,11 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>twitter-userstream-elasticsearch</artifactId>
+ <name>twitter-userstream-elasticsearch</name>
+
+ <description>
+ Connects to an active twitter account and stores the userstream as activities in Elasticsearch
+ </description>
<properties>
<elasticsearch.version>1.1.0</elasticsearch.version>
@@ -63,7 +68,6 @@
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-core</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
@@ -72,32 +76,26 @@
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-config</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-runtime-local</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-filters</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-provider-twitter</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-persist-elasticsearch</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/main/resources/TwitterUserstreamElasticsearch.dot.png
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/src/main/resources/TwitterUserstreamElasticsearch.dot.png b/local/twitter-userstream-elasticsearch/src/main/resources/TwitterUserstreamElasticsearch.dot.png
new file mode 100644
index 0000000..6e8713f
Binary files /dev/null and b/local/twitter-userstream-elasticsearch/src/main/resources/TwitterUserstreamElasticsearch.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/src/site/markdown/index.md b/local/twitter-userstream-elasticsearch/src/site/markdown/index.md
new file mode 100644
index 0000000..1ecf179
--- /dev/null
+++ b/local/twitter-userstream-elasticsearch/src/site/markdown/index.md
@@ -0,0 +1,73 @@
+Apache Streams (incubating)
+Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
+--------------------------------------------------------------------------------
+
+twitter-userstream-elasticsearch
+==============================
+
+Requirements:
+-------------
+ - Authorized Twitter API credentials
+ - A running ElasticSearch 1.0.0+ instance
+
+Description:
+------------
+This example connects to an active twitter account and stores the userstream as activities in Elasticsearch
+
+Specification:
+-----------------
+
+[TwitterUserstreamElasticsearch.dot](TwitterUserstreamElasticsearch.dot "TwitterUserstreamElasticsearch.dot" )
+
+Diagram:
+-----------------
+
+
+
+Example Configuration:
+----------------------
+
+ twitter {
+ endpoint = "userstream"
+ oauth {
+ consumerKey = "bcg14JThZEGoZ3MZOoT2HnJS7"
+ consumerSecret = "S4dwxnZni58CIJaoupGnUrO4HRHmbBGOb28W6IqOJBx36LPw2z"
+ accessToken = ""
+ accessTokenSecret = ""
+ }
+ }
+ elasticsearch {
+ hosts = [
+ localhost
+ ]
+ port = 9300
+ clusterName = elasticsearch
+ index = userstream_activity
+ type = activity
+ batchSize = 1
+ }
+
+The consumerKey and consumerSecret are set for our streams-example application
+The accessToken and accessTokenSecret can be obtained by navigating to:
+
+ https://api.twitter.com/oauth/authenticate?oauth_token=UIJ0AUxCJatpKDUyFt0OTSEP4asZgqxRwUCT0AMSwc&oauth_callback=http%3A%2F%2Foauth.streamstutorial.w2odata.com%3A8080%2Fsocialauthdemo%2FsocialAuthSuccessAction.do
+
+Build:
+---------
+
+`mvn clean package verify`
+
+Run:
+--------
+
+`java -cp target/twitter-userstream-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=src/main/resources/application.json org.apache.streams.example.twitter.TwitterUserstreamElasticsearch`
+
+Deploy:
+--------
+`mvn -Pdocker clean package docker:build`
+
+`docker tag twitter-userstream-elasticsearch:0.2-incubating-SNAPSHOT <dockerregistry>:twitter-userstream-elasticsearch:0.2-incubating-SNAPSHOT`
+
+`docker push <dockerregistry>:twitter-userstream-elasticsearch:0.2-incubating-SNAPSHOT`
+
+`docker run <dockerregistry>:twitter-userstream-elasticsearch:0.2-incubating-SNAPSHOT.jar java -cp twitter-userstream-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.twitter.TwitterUserstreamElasticsearch`
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot b/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot
new file mode 100644
index 0000000..c876176
--- /dev/null
+++ b/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot
@@ -0,0 +1,28 @@
+digraph g {
+
+ //providers
+ TwitterStreamProvider [label="TwitterStreamProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java"];
+
+ //processors
+ ActivityConverterProcessor [label="ActivityConverterProcessor",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterProcessor.java"];
+ DeleteOnlyProcessor [label="VerbDefinitionKeepFilter (verb:delete)",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionKeepFilter.java"];
+ NoDeletesProcessor [label="VerbDefinitionDropFilter (verb:delete)",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionDropFilter.java"];
+ SetDeleteIdProcessor [label="SetDeleteIdProcessor (verb:post)",shape=box,URL="https://github.com/apache/incubator-streams-examples/blob/master/local/twitter-userstream-elasticsearch/src/main/java/org/apache/streams/twitter/example/TwitterUserstreamElasticsearch.java"];
+
+ //persisters
+ ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"];
+ ElasticsearchPersistDeleter [label="ElasticsearchPersistDeleter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistDeleter.java"];
+
+ //data
+ es [label="es://{index}/{type}",shape=box];
+
+ //stream
+ TwitterStreamProvider -> ActivityConverterProcessor [label="ObjectNode"];
+ ActivityConverterProcessor -> DeleteOnlyProcessor [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+ ActivityConverterProcessor -> NoDeletesProcessor [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+ DeleteOnlyProcessor -> SetDeleteIdProcessor [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+ NoDeletesProcessor -> ElasticsearchPersistWriter [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+ ElasticsearchPersistWriter -> es [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"];
+ SetDeleteIdProcessor -> ElasticsearchPersistDeleter [label="Delete",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/verbs/delete.json"];
+ ElasticsearchPersistDeleter -> es [label="Delete",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/delete.json"];
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot.png
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot.png b/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot.png
new file mode 100644
index 0000000..6e8713f
Binary files /dev/null and b/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot.png differ
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/site/resources/application.conf
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/src/site/resources/application.conf b/local/twitter-userstream-elasticsearch/src/site/resources/application.conf
new file mode 100644
index 0000000..50d48b5
--- /dev/null
+++ b/local/twitter-userstream-elasticsearch/src/site/resources/application.conf
@@ -0,0 +1,22 @@
+twitter {
+ endpoint = "userstream"
+ oauth {
+ consumerKey = ""
+ consumerSecret = ""
+ accessToken = ""
+ accessTokenSecret = ""
+ }
+ follow = [
+
+ ]
+}
+elasticsearch {
+ hosts = [
+ localhost
+ ]
+ port = 9300
+ clusterName = elasticsearch
+ index = userstream_activity
+ type = activity
+ batchSize = 1
+}