You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/10/09 21:35:45 UTC
[4/6] incubator-streams-examples git commit: improve documentation
improve documentation
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/58fefc07
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/58fefc07
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/58fefc07
Branch: refs/heads/master
Commit: 58fefc07fc09b45f0e4cfebc9b126cab1fa8a9a3
Parents: 0112a83
Author: Steve Blackmon @steveblackmon <sb...@apache.org>
Authored: Thu Sep 29 22:02:14 2016 -0500
Committer: Steve Blackmon @steveblackmon <sb...@apache.org>
Committed: Thu Sep 29 22:02:14 2016 -0500
----------------------------------------------------------------------
.../resources/FlinkTwitterFollowingPipeline.dot | 37 ++++++++++++
.../FlinkTwitterFollowingPipeline.dot.svg | 63 ++++++++++++++++++++
.../resources/FlinkTwitterPostsPipeline.dot | 37 ++++++++++++
.../resources/FlinkTwitterPostsPipeline.dot.svg | 63 ++++++++++++++++++++
.../resources/FlinkTwitterSpritzerPipeline.dot | 33 ++++++++++
.../FlinkTwitterSpritzerPipeline.dot.svg | 47 +++++++++++++++
.../FlinkTwitterUserInformationPipeline.dot | 37 ++++++++++++
.../FlinkTwitterUserInformationPipeline.dot.svg | 63 ++++++++++++++++++++
.../markdown/FlinkTwitterFollowingPipeline.md | 4 +-
.../site/markdown/FlinkTwitterPostsPipeline.md | 2 +-
.../markdown/FlinkTwitterSpritzerPipeline.md | 41 +++++++++++++
.../FlinkTwitterUserInformationPipeline.md | 4 +-
.../src/site/markdown/index.md | 16 ++++-
.../test/FlinkTwitterFollowingPipelineIT.scala | 4 +-
14 files changed, 444 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot
new file mode 100644
index 0000000..ba5e60d
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ digraph g {
+
+ //source
+ source [label="source\nhdfs://${host}:${port}/${path}/${readerPath}",shape=tab];
+
+ //providers
+ TwitterFollowingProvider [label="TwitterFollowingProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProvider.java"];
+
+ //persisters
+ RollingFileSink [label="RollingFileSink",shape=ellipse];
+
+ //data
+ destination [label="destination\nhdfs://${host}:${port}/${path}/${writerPath}",shape=tab];
+
+ //stream
+ TwitterFollowingProvider -> source [dir=back,style=dashed];
+ TwitterFollowingProvider -> RollingFileSink [label="String"];
+ RollingFileSink -> destination;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot.svg
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot.svg b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot.svg
new file mode 100644
index 0000000..79bee38
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot.svg
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.39.20160214.2135 (20160214.2135)
+ -->
+<!-- Title: g Pages: 1 -->
+<svg width="504pt" height="203pt"
+ viewBox="0.00 0.00 504.29 203.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 199)">
+<title>g</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-199 500.286,-199 500.286,4 -4,4"/>
+<!-- source -->
+<g id="node1" class="node">
+<title>source</title>
+<polygon fill="none" stroke="black" points="275.082,-109 11.9727,-109 11.9727,-113 -0.0273492,-113 -0.0273492,-73 275.082,-73 275.082,-109"/>
+<polyline fill="none" stroke="black" points="-0.0273492,-109 11.9727,-109 "/>
+<text text-anchor="middle" x="137.527" y="-93.8" font-family="Times,serif" font-size="14.00">source</text>
+<text text-anchor="middle" x="137.527" y="-79.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${readerPath}</text>
+</g>
+<!-- TwitterFollowingProvider -->
+<g id="node2" class="node">
+<title>TwitterFollowingProvider</title>
+<g id="a_node2"><a xlink:href="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProvider.java" xlink:title="TwitterFollowingProvider">
+<ellipse fill="none" stroke="black" cx="200.527" cy="-177" rx="102.174" ry="18"/>
+<text text-anchor="middle" x="200.527" y="-172.8" font-family="Times,serif" font-size="14.00">TwitterFollowingProvider</text>
+</a>
+</g>
+</g>
+<!-- TwitterFollowingProvider->source -->
+<g id="edge1" class="edge">
+<title>TwitterFollowingProvider->source</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M181.043,-150.402C171.25,-137.034 159.635,-121.178 150.869,-109.212"/>
+<polygon fill="black" stroke="black" points="178.434,-152.763 187.167,-158.762 184.081,-148.626 178.434,-152.763"/>
+</g>
+<!-- RollingFileSink -->
+<g id="node3" class="node">
+<title>RollingFileSink</title>
+<ellipse fill="none" stroke="black" cx="359.527" cy="-91" rx="66.7358" ry="18"/>
+<text text-anchor="middle" x="359.527" y="-86.8" font-family="Times,serif" font-size="14.00">RollingFileSink</text>
+</g>
+<!-- TwitterFollowingProvider->RollingFileSink -->
+<g id="edge2" class="edge">
+<title>TwitterFollowingProvider->RollingFileSink</title>
+<path fill="none" stroke="black" d="M232.321,-159.803C257.806,-146.019 293.67,-126.621 320.824,-111.934"/>
+<polygon fill="black" stroke="black" points="322.54,-114.985 329.671,-107.149 319.21,-108.828 322.54,-114.985"/>
+<text text-anchor="middle" x="306.641" y="-129.8" font-family="Times,serif" font-size="14.00">String</text>
+</g>
+<!-- destination -->
+<g id="node4" class="node">
+<title>destination</title>
+<polygon fill="none" stroke="black" points="496.044,-36 235.01,-36 235.01,-40 223.01,-40 223.01,-0 496.044,-0 496.044,-36"/>
+<polyline fill="none" stroke="black" points="223.01,-36 235.01,-36 "/>
+<text text-anchor="middle" x="359.527" y="-20.8" font-family="Times,serif" font-size="14.00">destination</text>
+<text text-anchor="middle" x="359.527" y="-6.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${writerPath}</text>
+</g>
+<!-- RollingFileSink->destination -->
+<g id="edge3" class="edge">
+<title>RollingFileSink->destination</title>
+<path fill="none" stroke="black" d="M359.527,-72.9551C359.527,-64.8828 359.527,-55.1764 359.527,-46.1817"/>
+<polygon fill="black" stroke="black" points="363.027,-46.0903 359.527,-36.0904 356.027,-46.0904 363.027,-46.0903"/>
+</g>
+</g>
+</svg>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot
new file mode 100644
index 0000000..1092ff4
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ digraph g {
+
+ //source
+ source [label="source\nhdfs://${host}:${port}/${path}/${readerPath}",shape=tab];
+
+ //providers
+ TwitterTimelineProvider [label="TwitterTimelineProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java"];
+
+ //persisters
+ RollingFileSink [label="RollingFileSink",shape=ellipse];
+
+ //data
+ destination [label="destination\nhdfs://${host}:${port}/${path}/${writerPath}",shape=tab];
+
+ //stream
+ TwitterTimelineProvider -> source [dir=back,style=dashed];
+ TwitterTimelineProvider -> RollingFileSink [label="String"];
+ RollingFileSink -> destination;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot.svg
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot.svg b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot.svg
new file mode 100644
index 0000000..5698c45
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot.svg
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.39.20160214.2135 (20160214.2135)
+ -->
+<!-- Title: g Pages: 1 -->
+<svg width="504pt" height="203pt"
+ viewBox="0.00 0.00 504.29 203.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 199)">
+<title>g</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-199 500.286,-199 500.286,4 -4,4"/>
+<!-- source -->
+<g id="node1" class="node">
+<title>source</title>
+<polygon fill="none" stroke="black" points="275.082,-109 11.9727,-109 11.9727,-113 -0.0273492,-113 -0.0273492,-73 275.082,-73 275.082,-109"/>
+<polyline fill="none" stroke="black" points="-0.0273492,-109 11.9727,-109 "/>
+<text text-anchor="middle" x="137.527" y="-93.8" font-family="Times,serif" font-size="14.00">source</text>
+<text text-anchor="middle" x="137.527" y="-79.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${readerPath}</text>
+</g>
+<!-- TwitterTimelineProvider -->
+<g id="node2" class="node">
+<title>TwitterTimelineProvider</title>
+<g id="a_node2"><a xlink:href="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java" xlink:title="TwitterTimelineProvider">
+<ellipse fill="none" stroke="black" cx="200.527" cy="-177" rx="97.6831" ry="18"/>
+<text text-anchor="middle" x="200.527" y="-172.8" font-family="Times,serif" font-size="14.00">TwitterTimelineProvider</text>
+</a>
+</g>
+</g>
+<!-- TwitterTimelineProvider->source -->
+<g id="edge1" class="edge">
+<title>TwitterTimelineProvider->source</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M181.043,-150.402C171.25,-137.034 159.635,-121.178 150.869,-109.212"/>
+<polygon fill="black" stroke="black" points="178.434,-152.763 187.167,-158.762 184.081,-148.626 178.434,-152.763"/>
+</g>
+<!-- RollingFileSink -->
+<g id="node3" class="node">
+<title>RollingFileSink</title>
+<ellipse fill="none" stroke="black" cx="359.527" cy="-91" rx="66.7358" ry="18"/>
+<text text-anchor="middle" x="359.527" y="-86.8" font-family="Times,serif" font-size="14.00">RollingFileSink</text>
+</g>
+<!-- TwitterTimelineProvider->RollingFileSink -->
+<g id="edge2" class="edge">
+<title>TwitterTimelineProvider->RollingFileSink</title>
+<path fill="none" stroke="black" d="M232.321,-159.803C257.806,-146.019 293.67,-126.621 320.824,-111.934"/>
+<polygon fill="black" stroke="black" points="322.54,-114.985 329.671,-107.149 319.21,-108.828 322.54,-114.985"/>
+<text text-anchor="middle" x="306.641" y="-129.8" font-family="Times,serif" font-size="14.00">String</text>
+</g>
+<!-- destination -->
+<g id="node4" class="node">
+<title>destination</title>
+<polygon fill="none" stroke="black" points="496.044,-36 235.01,-36 235.01,-40 223.01,-40 223.01,-0 496.044,-0 496.044,-36"/>
+<polyline fill="none" stroke="black" points="223.01,-36 235.01,-36 "/>
+<text text-anchor="middle" x="359.527" y="-20.8" font-family="Times,serif" font-size="14.00">destination</text>
+<text text-anchor="middle" x="359.527" y="-6.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${writerPath}</text>
+</g>
+<!-- RollingFileSink->destination -->
+<g id="edge3" class="edge">
+<title>RollingFileSink->destination</title>
+<path fill="none" stroke="black" d="M359.527,-72.9551C359.527,-64.8828 359.527,-55.1764 359.527,-46.1817"/>
+<polygon fill="black" stroke="black" points="363.027,-46.0903 359.527,-36.0904 356.027,-46.0904 363.027,-46.0903"/>
+</g>
+</g>
+</svg>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot
new file mode 100644
index 0000000..5a57595
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ digraph g {
+
+ //providers
+ TwitterStreamProvider [label="TwitterStreamProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java"];
+
+ //persisters
+ RollingFileSink [label="RollingFileSink",shape=ellipse];
+
+ //data
+ destination [label="hdfs://${host}:${port}/${path}/${writerPath}",shape=box];
+
+ //stream
+ TwitterStreamProvider -> RollingFileSink [label="String"];
+ RollingFileSink -> destination;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot.svg
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot.svg b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot.svg
new file mode 100644
index 0000000..960a11f
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot.svg
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.39.20160214.2135 (20160214.2135)
+ -->
+<!-- Title: g Pages: 1 -->
+<svg width="282pt" height="203pt"
+ viewBox="0.00 0.00 281.52 203.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 199)">
+<title>g</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-199 277.517,-199 277.517,4 -4,4"/>
+<!-- TwitterStreamProvider -->
+<g id="node1" class="node">
+<title>TwitterStreamProvider</title>
+<g id="a_node1"><a xlink:href="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java" xlink:title="TwitterStreamProvider">
+<ellipse fill="none" stroke="black" cx="136.758" cy="-177" rx="91.0473" ry="18"/>
+<text text-anchor="middle" x="136.758" y="-172.8" font-family="Times,serif" font-size="14.00">TwitterStreamProvider</text>
+</a>
+</g>
+</g>
+<!-- RollingFileSink -->
+<g id="node2" class="node">
+<title>RollingFileSink</title>
+<ellipse fill="none" stroke="black" cx="136.758" cy="-91" rx="66.7358" ry="18"/>
+<text text-anchor="middle" x="136.758" y="-86.8" font-family="Times,serif" font-size="14.00">RollingFileSink</text>
+</g>
+<!-- TwitterStreamProvider->RollingFileSink -->
+<g id="edge1" class="edge">
+<title>TwitterStreamProvider->RollingFileSink</title>
+<path fill="none" stroke="black" d="M136.758,-158.762C136.758,-147.36 136.758,-132.434 136.758,-119.494"/>
+<polygon fill="black" stroke="black" points="140.258,-119.212 136.758,-109.212 133.258,-119.212 140.258,-119.212"/>
+<text text-anchor="middle" x="153.872" y="-129.8" font-family="Times,serif" font-size="14.00">String</text>
+</g>
+<!-- destination -->
+<g id="node3" class="node">
+<title>destination</title>
+<polygon fill="none" stroke="black" points="273.275,-36 0.241273,-36 0.241273,-0 273.275,-0 273.275,-36"/>
+<text text-anchor="middle" x="136.758" y="-13.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${writerPath}</text>
+</g>
+<!-- RollingFileSink->destination -->
+<g id="edge2" class="edge">
+<title>RollingFileSink->destination</title>
+<path fill="none" stroke="black" d="M136.758,-72.9551C136.758,-64.8828 136.758,-55.1764 136.758,-46.1817"/>
+<polygon fill="black" stroke="black" points="140.258,-46.0903 136.758,-36.0904 133.258,-46.0904 140.258,-46.0903"/>
+</g>
+</g>
+</svg>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot
new file mode 100644
index 0000000..4a37234
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ digraph g {
+
+ //source
+ source [label="source\nhdfs://${host}:${port}/${path}/${readerPath}",shape=tab];
+
+ //providers
+ TwitterUserInformationProvider [label="TwitterUserInformationProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProvider.java"];
+
+ //persisters
+ RollingFileSink [label="RollingFileSink",shape=ellipse];
+
+ //data
+ destination [label="destination\nhdfs://${host}:${port}/${path}/${writerPath}",shape=tab];
+
+ //stream
+ TwitterUserInformationProvider -> source [dir=back,style=dashed];
+ TwitterUserInformationProvider -> RollingFileSink [label="String"];
+ RollingFileSink -> destination;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot.svg
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot.svg b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot.svg
new file mode 100644
index 0000000..9dadc63
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot.svg
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.39.20160214.2135 (20160214.2135)
+ -->
+<!-- Title: g Pages: 1 -->
+<svg width="504pt" height="203pt"
+ viewBox="0.00 0.00 504.29 203.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 199)">
+<title>g</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-199 500.286,-199 500.286,4 -4,4"/>
+<!-- source -->
+<g id="node1" class="node">
+<title>source</title>
+<polygon fill="none" stroke="black" points="275.082,-109 11.9727,-109 11.9727,-113 -0.0273492,-113 -0.0273492,-73 275.082,-73 275.082,-109"/>
+<polyline fill="none" stroke="black" points="-0.0273492,-109 11.9727,-109 "/>
+<text text-anchor="middle" x="137.527" y="-93.8" font-family="Times,serif" font-size="14.00">source</text>
+<text text-anchor="middle" x="137.527" y="-79.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${readerPath}</text>
+</g>
+<!-- TwitterUserInformationProvider -->
+<g id="node2" class="node">
+<title>TwitterUserInformationProvider</title>
+<g id="a_node2"><a xlink:href="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProvider.java" xlink:title="TwitterUserInformationProvider">
+<ellipse fill="none" stroke="black" cx="200.527" cy="-177" rx="124.824" ry="18"/>
+<text text-anchor="middle" x="200.527" y="-172.8" font-family="Times,serif" font-size="14.00">TwitterUserInformationProvider</text>
+</a>
+</g>
+</g>
+<!-- TwitterUserInformationProvider->source -->
+<g id="edge1" class="edge">
+<title>TwitterUserInformationProvider->source</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M181.043,-150.402C171.25,-137.034 159.635,-121.178 150.869,-109.212"/>
+<polygon fill="black" stroke="black" points="178.434,-152.763 187.167,-158.762 184.081,-148.626 178.434,-152.763"/>
+</g>
+<!-- RollingFileSink -->
+<g id="node3" class="node">
+<title>RollingFileSink</title>
+<ellipse fill="none" stroke="black" cx="359.527" cy="-91" rx="66.7358" ry="18"/>
+<text text-anchor="middle" x="359.527" y="-86.8" font-family="Times,serif" font-size="14.00">RollingFileSink</text>
+</g>
+<!-- TwitterUserInformationProvider->RollingFileSink -->
+<g id="edge2" class="edge">
+<title>TwitterUserInformationProvider->RollingFileSink</title>
+<path fill="none" stroke="black" d="M232.703,-159.597C258.14,-145.839 293.738,-126.584 320.747,-111.975"/>
+<polygon fill="black" stroke="black" points="322.42,-115.05 329.551,-107.214 319.09,-108.893 322.42,-115.05"/>
+<text text-anchor="middle" x="306.641" y="-129.8" font-family="Times,serif" font-size="14.00">String</text>
+</g>
+<!-- destination -->
+<g id="node4" class="node">
+<title>destination</title>
+<polygon fill="none" stroke="black" points="496.044,-36 235.01,-36 235.01,-40 223.01,-40 223.01,-0 496.044,-0 496.044,-36"/>
+<polyline fill="none" stroke="black" points="223.01,-36 235.01,-36 "/>
+<text text-anchor="middle" x="359.527" y="-20.8" font-family="Times,serif" font-size="14.00">destination</text>
+<text text-anchor="middle" x="359.527" y="-6.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${writerPath}</text>
+</g>
+<!-- RollingFileSink->destination -->
+<g id="edge3" class="edge">
+<title>RollingFileSink->destination</title>
+<path fill="none" stroke="black" d="M359.527,-72.9551C359.527,-64.8828 359.527,-55.1764 359.527,-46.1817"/>
+<polygon fill="black" stroke="black" points="363.027,-46.0903 359.527,-36.0904 356.027,-46.0904 363.027,-46.0903"/>
+</g>
+</g>
+</svg>
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md
index 22f30f5..3ad23d3 100644
--- a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md
+++ b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md
@@ -19,7 +19,9 @@ Diagram:
Example Configuration:
----------------------
-[FlinkTwitterFollowingPipeline.json](FlinkTwitterFollowingPipeline.json "FlinkTwitterFollowingPipeline.json" )
+[FlinkTwitterFollowingPipelineFollowersIT.conf](FlinkTwitterFollowingPipelineFollowersIT.conf "FlinkTwitterFollowingPipelineFollowersIT.conf" )
+
+[FlinkTwitterFollowingPipelineFriendsIT.conf](FlinkTwitterFollowingPipelineFriendsIT.conf "FlinkTwitterFollowingPipelineFriendsIT.conf" )
Run (Local):
------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md
index 5f77994..fe6b544 100644
--- a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md
+++ b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md
@@ -19,7 +19,7 @@ Diagram:
Example Configuration:
----------------------
-[FlinkTwitterPostsPipeline.json](FlinkTwitterPostsPipeline.json "FlinkTwitterPostsPipeline.json" )
+[FlinkTwitterPostsPipelineIT.conf](FlinkTwitterPostsPipelineIT.conf "FlinkTwitterPostsPipelineIT.conf" )
Run (Local):
------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterSpritzerPipeline.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterSpritzerPipeline.md b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterSpritzerPipeline.md
new file mode 100644
index 0000000..259fe7f
--- /dev/null
+++ b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterSpritzerPipeline.md
@@ -0,0 +1,41 @@
+FlinkTwitterSpritzerPipeline
+============================
+
+Description:
+-----------------
+
+Collects twitter posts in real-time from the sample endpoint with flink.
+
+Specification:
+-----------------
+
+[FlinkTwitterSpritzerPipeline.dot](FlinkTwitterSpritzerPipeline.dot "FlinkTwitterSpritzerPipeline.dot" )
+
+Diagram:
+-----------------
+
+
+
+Example Configuration:
+----------------------
+
+[FlinkTwitterSpritzerPipelineIT.conf](FlinkTwitterSpritzerPipelineIT.conf "FlinkTwitterSpritzerPipelineIT.conf" )
+
+Run (Local):
+------------
+
+ java -cp dist/flink-twitter-collection-jar-with-dependencies.jar -Dconfig.file=file://<location_of_config_file> org.apache.streams.examples.flink.twitter.collection.FlinkTwitterPostsPipeline
+
+Run (Flink):
+------------
+
+ flink-run.sh dist/flink-twitter-collection-jar-with-dependencies.jar org.apache.streams.examples.flink.twitter.collection.FlinkTwitterPostsPipeline http://<location_of_config_file>
+
+Run (YARN):
+-----------
+
+ flink-run.sh yarn dist/flink-twitter-collection-jar-with-dependencies.jar org.apache.streams.examples.flink.twitter.collection.FlinkTwitterPostsPipeline http://<location_of_config_file>
+
+[JavaDocs](apidocs/index.html "JavaDocs")
+
+###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md
index 5e0d1fe..a465de9 100644
--- a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md
+++ b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md
@@ -4,7 +4,7 @@ FlinkTwitterUserInformationPipeline
Description:
-----------------
-Collects twitter users with flink.
+Collects twitter user profiles with flink.
Specification:
-----------------
@@ -19,7 +19,7 @@ Diagram:
Example Configuration:
----------------------
-[FlinkTwitterUserInformationPipeline.json](FlinkTwitterUserInformationPipeline.json "FlinkTwitterUserInformationPipeline.json" )
+[FlinkTwitterUserInformationPipelineIT.conf](FlinkTwitterUserInformationPipelineIT.conf "FlinkTwitterUserInformationPipelineIT.conf" )
Run (Local):
------------
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/index.md b/flink/flink-twitter-collection/src/site/markdown/index.md
index 19e44cf..0f15603 100644
--- a/flink/flink-twitter-collection/src/site/markdown/index.md
+++ b/flink/flink-twitter-collection/src/site/markdown/index.md
@@ -22,10 +22,24 @@ Streams:
<a href="FlinkTwitterFollowingPipeline.html" target="_self">FlinkTwitterFollowingPipeline</a>
+Test:
+-----
+
+Create a local file `application.conf` with valid twitter credentials
+
+ twitter {
+ oauth {
+ consumerKey = ""
+ consumerSecret = ""
+ accessToken = ""
+ accessTokenSecret = ""
+ }
+ }
+
Build:
---------
- mvn clean install verify
+ mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf"
[JavaDocs](apidocs/index.html "JavaDocs")
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala
index b051e90..e6294f6 100644
--- a/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala
+++ b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala
@@ -27,7 +27,7 @@ class FlinkTwitterFollowingPipelineIT extends FlatSpec {
import FlinkTwitterFollowingPipeline._
- @Test
+ @Test(enabled = false)
def flinkTwitterFollowersPipelineFriendsIT = {
val reference: Config = ConfigFactory.load()
@@ -55,7 +55,7 @@ class FlinkTwitterFollowingPipelineIT extends FlatSpec {
}
- @Test
+ @Test(enabled = false)
def flinkTwitterFollowersPipelineFollowersIT = {
val reference: Config = ConfigFactory.load()