You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by me...@apache.org on 2018/08/20 21:40:43 UTC
[beam-site] 06/11: Updates to streaming connectors blog post
This is an automated email from the ASF dual-hosted git repository.
mergebot-role pushed a commit to branch mergebot
in repository https://gitbox.apache.org/repos/asf/beam-site.git
commit 3cd63cdd4666bf38dfbd5448dd40155b4d6f6015
Author: Julien Phalip <jp...@gmail.com>
AuthorDate: Tue Aug 14 11:23:28 2018 -0400
Updates to streaming connectors blog post
---
...2018-08-XX-review-input-streaming-connectors.md | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/src/_posts/2018-08-XX-review-input-streaming-connectors.md b/src/_posts/2018-08-XX-review-input-streaming-connectors.md
index fded813..5816292 100644
--- a/src/_posts/2018-08-XX-review-input-streaming-connectors.md
+++ b/src/_posts/2018-08-XX-review-input-streaming-connectors.md
@@ -41,7 +41,7 @@ Below are the main streaming input connectors for available for Beam and Spark D
</td>
<td>Local<br>(Using the <code>file://</code> URI)
</td>
- <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.5.0/org/apache/beam/sdk/io/TextIO.html">TextIO</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/io/TextIO.html">TextIO</a>
</td>
<td><a href="https://spark.apache.org/docs/latest/api/java/org/apache/spark/streaming/StreamingContext.html#textFileStream-java.lang.String-">textFileStream</a><br>(Spark treats most Unix systems as HDFS-compatible, but the location should be accessible from all nodes)
</td>
@@ -49,7 +49,7 @@ Below are the main streaming input connectors for available for Beam and Spark D
<tr>
<td>HDFS<br>(Using the <code>hdfs://</code> URI)
</td>
- <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.5.0/org/apache/beam/sdk/io/FileIO.html">FileIO</a> + <a href="https://beam.apache.org/documentation/sdks/javadoc/2.5.0/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.html">HadoopFileSystemOptions</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/io/FileIO.html">FileIO</a> + <a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.html">HadoopFileSystemOptions</a>
</td>
<td><a href="https://spark.apache.org/docs/latest/api/java/org/apache/spark/streaming/util/HdfsUtils.html">HdfsUtils</a>
</td>
@@ -59,7 +59,7 @@ Below are the main streaming input connectors for available for Beam and Spark D
</td>
<td>Cloud Storage<br>(Using the <code>gs://</code> URI)
</td>
- <td rowspan="2" ><a href="https://beam.apache.org/documentation/sdks/javadoc/2.5.0/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.html">HadoopFileSystemOptions</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/io/FileIO.html">FileIO</a> + <a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/extensions/gcp/options/GcsOptions.html">GcsOptions</a>
</td>
<td rowspan="2" ><a href="https://spark.apache.org/docs/latest/api/java/org/apache/spark/SparkContext.html#hadoopConfiguration--">hadoopConfiguration</a>
<p>
@@ -69,13 +69,15 @@ and <a href="https://spark.apache.org/docs/latest/api/java/org/apache/spark/stre
<tr>
<td>S3<br>(Using the <code>s3://</code> URI)
</td>
+ <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/io/FileIO.html">FileIO</a> + <a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/io/aws/options/S3Options.html">S3Options</a>
+ </td>
</tr>
<tr>
<td rowspan="3" >Messaging Queues
</td>
<td>Kafka
</td>
- <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.5.0/org/apache/beam/sdk/io/kafka/KafkaIO.html">KafkaIO</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/io/kafka/KafkaIO.html">KafkaIO</a>
</td>
<td><a href="https://spark.apache.org/docs/latest/streaming-kafka-0-10-integration.html">spark-streaming-kafka</a>
</td>
@@ -83,7 +85,7 @@ and <a href="https://spark.apache.org/docs/latest/api/java/org/apache/spark/stre
<tr>
<td>Kinesis
</td>
- <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.5.0/org/apache/beam/sdk/io/kinesis/KinesisIO.html">KinesisIO</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/io/kinesis/KinesisIO.html">KinesisIO</a>
</td>
<td><a href="https://spark.apache.org/docs/latest/streaming-kinesis-integration.html">spark-streaming-kinesis</a>
</td>
@@ -91,7 +93,7 @@ and <a href="https://spark.apache.org/docs/latest/api/java/org/apache/spark/stre
<tr>
<td>Cloud Pub/Sub
</td>
- <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.5.0/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.html">PubsubIO</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/javadoc/2.6.0/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.html">PubsubIO</a>
</td>
<td><a href="https://github.com/apache/bahir/tree/master/streaming-pubsub">spark-streaming-pubsub</a> from <a href="http://bahir.apache.org">Apache Bahir</a>
</td>
@@ -132,7 +134,7 @@ Below are the main streaming input connectors for available for Beam and Spark D
</td>
<td>Local
</td>
- <td><a href="https://beam.apache.org/documentation/sdks/pydoc/2.5.0/apache_beam.io.textio.html">io.textio</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/pydoc/2.6.0/apache_beam.io.textio.html">io.textio</a>
</td>
<td><a href="http://spark.apache.org/docs/latest/api/python/pyspark.streaming.html#pyspark.streaming.StreamingContext.textFileStream">textFileStream</a>
</td>
@@ -140,7 +142,7 @@ Below are the main streaming input connectors for available for Beam and Spark D
<tr>
<td>HDFS
</td>
- <td><a href="https://beam.apache.org/documentation/sdks/pydoc/2.5.0/apache_beam.io.hadoopfilesystem.html">io.hadoopfilesystem</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/pydoc/2.6.0/apache_beam.io.hadoopfilesystem.html">io.hadoopfilesystem</a>
</td>
<td><a href="https://spark.apache.org/docs/latest/api/java/org/apache/spark/SparkContext.html#hadoopConfiguration--">hadoopConfiguration</a> (Access through <code>sc._jsc</code> with Py4J)
and <a href="http://spark.apache.org/docs/latest/api/python/pyspark.streaming.html#pyspark.streaming.StreamingContext.textFileStream">textFileStream</a>
@@ -151,7 +153,7 @@ and <a href="http://spark.apache.org/docs/latest/api/python/pyspark.streaming.ht
</td>
<td>Google Cloud Storage
</td>
- <td><a href="https://beam.apache.org/documentation/sdks/pydoc/2.5.0/apache_beam.io.gcp.gcsio.html">io.gcp.gcsio</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/pydoc/2.6.0/apache_beam.io.gcp.gcsio.html">io.gcp.gcsio</a>
</td>
<td rowspan="2" ><a href="http://spark.apache.org/docs/latest/api/python/pyspark.streaming.html#pyspark.streaming.StreamingContext.textFileStream">textFileStream</a>
</td>
@@ -183,7 +185,7 @@ and <a href="http://spark.apache.org/docs/latest/api/python/pyspark.streaming.ht
<tr>
<td>Cloud Pub/Sub
</td>
- <td><a href="https://beam.apache.org/documentation/sdks/pydoc/2.5.0/apache_beam.io.gcp.pubsub.html">io.gcp.pubsub</a>
+ <td><a href="https://beam.apache.org/documentation/sdks/pydoc/2.6.0/apache_beam.io.gcp.pubsub.html">io.gcp.pubsub</a>
</td>
<td>N/A
</td>