You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/10/11 21:17:11 UTC
[6/7] incubator-streams-examples git commit: updates to docs and poms
updates to docs and poms
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/476d356f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/476d356f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/476d356f
Branch: refs/heads/master
Commit: 476d356f05a6e07a935281ee4cdc54ac91e48de9
Parents: 4a70f9a
Author: Steve Blackmon @steveblackmon <sb...@apache.org>
Authored: Mon Oct 10 17:25:42 2016 -0500
Committer: Steve Blackmon @steveblackmon <sb...@apache.org>
Committed: Mon Oct 10 17:25:42 2016 -0500
----------------------------------------------------------------------
local/elasticsearch-hdfs/pom.xml | 4 --
.../src/site/markdown/index.md | 18 +++++--
.../src/test/resources/ElasticsearchHdfsIT.conf | 15 ++++++
.../src/test/resources/HdfsElasticsearchIT.conf | 15 ++++++
local/elasticsearch-reindex/pom.xml | 4 --
.../src/site/markdown/index.md | 10 +++-
.../resources/ElasticsearchReindexChildIT.conf | 18 +++++--
.../test/resources/ElasticsearchReindexIT.conf | 18 +++++--
.../resources/ElasticsearchReindexParentIT.conf | 18 +++++--
local/mongo-elasticsearch-sync/pom.xml | 4 --
.../src/site/markdown/index.md | 28 ++++++++++-
.../resources/MongoElasticsearchSyncIT.conf | 50 ++++++++++++--------
local/twitter-history-elasticsearch/pom.xml | 5 --
.../src/site/markdown/index.md | 8 +++-
.../TwitterHistoryElasticsearchIT.conf | 21 +++++---
.../src/site/markdown/index.md | 10 +++-
.../TwitterUserstreamElasticsearchIT.conf | 15 ++++++
17 files changed, 202 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/elasticsearch-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/pom.xml b/local/elasticsearch-hdfs/pom.xml
index 601cd2b..99458b1 100644
--- a/local/elasticsearch-hdfs/pom.xml
+++ b/local/elasticsearch-hdfs/pom.xml
@@ -303,10 +303,6 @@
</dependency>
</dependencies>
</plugin>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/elasticsearch-hdfs/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/site/markdown/index.md b/local/elasticsearch-hdfs/src/site/markdown/index.md
index e54924f..b014a19 100644
--- a/local/elasticsearch-hdfs/src/site/markdown/index.md
+++ b/local/elasticsearch-hdfs/src/site/markdown/index.md
@@ -19,10 +19,22 @@ Streams:
Build:
---------
- mvn clean install verify
+ mvn clean install
-Note that an alternative version of hdfs is packaged, by excluding org.apache.hadoop.hadoop-hdfs when
-importing org.apache.streams.streams-persist-hdfs, and specifically depending on a different preferred version.
+Testing:
+---------
+
+Start up elasticsearch with docker:
+
+ mvn -PdockerITs docker:start
+
+Build with integration testing enabled, using your credentials
+
+ mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf"
+
+Shutdown elasticsearch when finished:
+
+ mvn -PdockerITs docker:stop
Deploy (Docker):
----------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/elasticsearch-hdfs/src/test/resources/ElasticsearchHdfsIT.conf
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/test/resources/ElasticsearchHdfsIT.conf b/local/elasticsearch-hdfs/src/test/resources/ElasticsearchHdfsIT.conf
index 0505876..80ef53a 100644
--- a/local/elasticsearch-hdfs/src/test/resources/ElasticsearchHdfsIT.conf
+++ b/local/elasticsearch-hdfs/src/test/resources/ElasticsearchHdfsIT.conf
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
source {
hosts += ${es.tcp.host}
port = ${es.tcp.port}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/elasticsearch-hdfs/src/test/resources/HdfsElasticsearchIT.conf
----------------------------------------------------------------------
diff --git a/local/elasticsearch-hdfs/src/test/resources/HdfsElasticsearchIT.conf b/local/elasticsearch-hdfs/src/test/resources/HdfsElasticsearchIT.conf
index 35b17a0..072a024 100644
--- a/local/elasticsearch-hdfs/src/test/resources/HdfsElasticsearchIT.conf
+++ b/local/elasticsearch-hdfs/src/test/resources/HdfsElasticsearchIT.conf
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
source {
fields = ["ID","DOC"]
scheme = file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/elasticsearch-reindex/pom.xml
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/pom.xml b/local/elasticsearch-reindex/pom.xml
index d4c5830..4678c42 100644
--- a/local/elasticsearch-reindex/pom.xml
+++ b/local/elasticsearch-reindex/pom.xml
@@ -267,10 +267,6 @@
</dependency>
</dependencies>
</plugin>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/elasticsearch-reindex/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/site/markdown/index.md b/local/elasticsearch-reindex/src/site/markdown/index.md
index 6383281..8aa394d 100644
--- a/local/elasticsearch-reindex/src/site/markdown/index.md
+++ b/local/elasticsearch-reindex/src/site/markdown/index.md
@@ -31,8 +31,16 @@ Populate source and destination in configuration with cluster / index / type det
Testing:
---------
+Start up elasticsearch with docker:
+
mvn -PdockerITs docker:start
- mvn clean install test verify -DskipITs=false
+
+Build with integration testing enabled, using your credentials
+
+ mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf"
+
+Shutdown elasticsearch when finished:
+
mvn -PdockerITs docker:stop
Run (Local):
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexChildIT.conf
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexChildIT.conf b/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexChildIT.conf
index 936d6df..424d725 100644
--- a/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexChildIT.conf
+++ b/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexChildIT.conf
@@ -1,7 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
"source": {
hosts += ${es.tcp.host}
port = ${es.tcp.port}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexIT.conf
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexIT.conf b/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexIT.conf
index 0b69948..0062f0f 100644
--- a/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexIT.conf
+++ b/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexIT.conf
@@ -1,7 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
"source": {
hosts += ${es.tcp.host}
port = ${es.tcp.port}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexParentIT.conf
----------------------------------------------------------------------
diff --git a/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexParentIT.conf b/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexParentIT.conf
index 936d6df..424d725 100644
--- a/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexParentIT.conf
+++ b/local/elasticsearch-reindex/src/test/resources/ElasticsearchReindexParentIT.conf
@@ -1,7 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
"source": {
hosts += ${es.tcp.host}
port = ${es.tcp.port}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/mongo-elasticsearch-sync/pom.xml
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/pom.xml b/local/mongo-elasticsearch-sync/pom.xml
index 2dde464..2a89e0d 100644
--- a/local/mongo-elasticsearch-sync/pom.xml
+++ b/local/mongo-elasticsearch-sync/pom.xml
@@ -271,10 +271,6 @@
</dependency>
</dependencies>
</plugin>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/mongo-elasticsearch-sync/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/src/site/markdown/index.md b/local/mongo-elasticsearch-sync/src/site/markdown/index.md
index 10bad65..42512b6 100644
--- a/local/mongo-elasticsearch-sync/src/site/markdown/index.md
+++ b/local/mongo-elasticsearch-sync/src/site/markdown/index.md
@@ -32,7 +32,33 @@ Example Configuration:
Build:
---------
- mvn clean package verify
+ mvn clean package
+
+Testing:
+---------
+
+Create a local file `application.conf` with valid twitter credentials
+
+ twitter {
+ oauth {
+ consumerKey = ""
+ consumerSecret = ""
+ accessToken = ""
+ accessTokenSecret = ""
+ }
+ }
+
+Start up elasticsearch and mongodb with docker:
+
+ mvn -PdockerITs docker:start
+
+Build with integration testing enabled, using your credentials
+
+ mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf"
+
+Shutdown elasticsearch and mongodb when finished:
+
+ mvn -PdockerITs docker:stop
Run (Local):
------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/mongo-elasticsearch-sync/src/test/resources/MongoElasticsearchSyncIT.conf
----------------------------------------------------------------------
diff --git a/local/mongo-elasticsearch-sync/src/test/resources/MongoElasticsearchSyncIT.conf b/local/mongo-elasticsearch-sync/src/test/resources/MongoElasticsearchSyncIT.conf
index c178e8f..61e61d7 100644
--- a/local/mongo-elasticsearch-sync/src/test/resources/MongoElasticsearchSyncIT.conf
+++ b/local/mongo-elasticsearch-sync/src/test/resources/MongoElasticsearchSyncIT.conf
@@ -1,21 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "source": {
- host = ${mongo.tcp.host}
- port = ${mongo.tcp.port}
- "db": "mongo_persist_it",
- "collection": "activity"
- },
- "destination": {
- hosts = [
- ${es.tcp.host}
- ]
- port = ${es.tcp.port}
- "clusterName": "elasticsearch",
- "index": "mongo_elasticsearch_sync_it",
- "type": "activity",
- "forceUseConfig": true
- }
+ "source": {
+ host = ${mongo.tcp.host}
+ port = ${mongo.tcp.port}
+ "db": "mongo_persist_it",
+ "collection": "activity"
+ },
+ "destination": {
+ hosts = [
+ ${es.tcp.host}
+ ]
+ port = ${es.tcp.port}
+ "clusterName": "elasticsearch",
+ "index": "mongo_elasticsearch_sync_it",
+ "type": "activity",
+ "forceUseConfig": true
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/twitter-history-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/pom.xml b/local/twitter-history-elasticsearch/pom.xml
index e7338df..1a96fb9 100644
--- a/local/twitter-history-elasticsearch/pom.xml
+++ b/local/twitter-history-elasticsearch/pom.xml
@@ -279,11 +279,6 @@
</dependency>
</dependencies>
</plugin>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- <version>${docker.plugin.version}</version>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/twitter-history-elasticsearch/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/site/markdown/index.md b/local/twitter-history-elasticsearch/src/site/markdown/index.md
index c3571e5..22eee99 100644
--- a/local/twitter-history-elasticsearch/src/site/markdown/index.md
+++ b/local/twitter-history-elasticsearch/src/site/markdown/index.md
@@ -50,10 +50,16 @@ Create a local file `application.conf` with valid twitter credentials
}
}
+Start up elasticsearch with docker:
+
+ mvn -PdockerITs docker:start
+
Build with integration testing enabled, using your credentials
- mvn -PdockerITs docker:start
mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf"
+
+Shutdown elasticsearch when finished:
+
mvn -PdockerITs docker:stop
Run (Local):
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/twitter-history-elasticsearch/src/test/resources/TwitterHistoryElasticsearchIT.conf
----------------------------------------------------------------------
diff --git a/local/twitter-history-elasticsearch/src/test/resources/TwitterHistoryElasticsearchIT.conf b/local/twitter-history-elasticsearch/src/test/resources/TwitterHistoryElasticsearchIT.conf
index 1cafc9f..9035ca3 100644
--- a/local/twitter-history-elasticsearch/src/test/resources/TwitterHistoryElasticsearchIT.conf
+++ b/local/twitter-history-elasticsearch/src/test/resources/TwitterHistoryElasticsearchIT.conf
@@ -1,10 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
twitter {
- oauth {
- consumerKey = "jF3awfLECUZ4tFAwS5bZcha8c"
- consumerSecret = "0IjoS5aPE88kNSREK6HNzAhUcJMziSlaT1fOkA5pzpusZLrhCj"
- accessToken = "42232950-CzaYlt2M6SPGI883B5NZ8vROcL4qUsTJlp9wIM2K2"
- accessTokenSecret = "vviQzladFUl23hdVelEiIknSLoHfAs40DqTv3RdXHhmz0"
- }
info = [
18055613
]
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/twitter-userstream-elasticsearch/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/src/site/markdown/index.md b/local/twitter-userstream-elasticsearch/src/site/markdown/index.md
index a10846c..f5379c9 100644
--- a/local/twitter-userstream-elasticsearch/src/site/markdown/index.md
+++ b/local/twitter-userstream-elasticsearch/src/site/markdown/index.md
@@ -48,13 +48,19 @@ Create a local file `application.conf` with valid twitter credentials
accessTokenSecret = ""
}
}
+
+Start up elasticsearch with docker:
+ mvn -PdockerITs docker:start
+
Build with integration testing enabled, using your credentials
- mvn -PdockerITs docker:start
mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf"
+
+Shutdown elasticsearch when finished:
+
mvn -PdockerITs docker:stop
-
+
Run (Local):
------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/476d356f/local/twitter-userstream-elasticsearch/src/test/resources/TwitterUserstreamElasticsearchIT.conf
----------------------------------------------------------------------
diff --git a/local/twitter-userstream-elasticsearch/src/test/resources/TwitterUserstreamElasticsearchIT.conf b/local/twitter-userstream-elasticsearch/src/test/resources/TwitterUserstreamElasticsearchIT.conf
index ae3b463..df9be4d 100644
--- a/local/twitter-userstream-elasticsearch/src/test/resources/TwitterUserstreamElasticsearchIT.conf
+++ b/local/twitter-userstream-elasticsearch/src/test/resources/TwitterUserstreamElasticsearchIT.conf
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
twitter {
endpoint = sample
track = [