You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2020/03/29 21:31:32 UTC
[spark] branch branch-2.4 updated:
[SPARK-31293][DSTREAMS][KINESIS][DOC] Fix wrong examples and help messages
for Kinesis integration
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new f05ac28a [SPARK-31293][DSTREAMS][KINESIS][DOC] Fix wrong examples and help messages for Kinesis integration
f05ac28a is described below
commit f05ac28a949b9ad5b154ac959b9da6a24a4e09e6
Author: Kengo Seki <se...@apache.org>
AuthorDate: Sun Mar 29 14:27:19 2020 -0700
[SPARK-31293][DSTREAMS][KINESIS][DOC] Fix wrong examples and help messages for Kinesis integration
This PR (SPARK-31293) fixes wrong command examples, parameter descriptions and help message format for Amazon Kinesis integration with Spark Streaming.
To improve usability of those commands.
No
I ran the fixed commands manually and confirmed they worked as expected.
Closes #28063 from sekikn/SPARK-31293.
Authored-by: Kengo Seki <se...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit 60dd1a690fed62b1d6442cdc8cf3f89ef4304d5a)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
docs/streaming-kinesis-integration.md | 3 +--
.../main/python/examples/streaming/kinesis_wordcount_asl.py | 7 ++++---
.../apache/spark/examples/streaming/KinesisWordCountASL.scala | 10 +++++-----
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md
index 6a52e8a..0685cd8 100644
--- a/docs/streaming-kinesis-integration.md
+++ b/docs/streaming-kinesis-integration.md
@@ -217,8 +217,7 @@ To run the example,
</div>
<div data-lang="python" markdown="1">
- bin/spark-submit --jars external/kinesis-asl/target/scala-*/\
- spark-streaming-kinesis-asl-assembly_*.jar \
+ ./bin/spark-submit --jars 'external/kinesis-asl-assembly/target/spark-streaming-kinesis-asl-assembly_*.jar' \
external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
[Kinesis app name] [Kinesis stream name] [endpoint URL] [region name]
diff --git a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
index 49794fa..777a332 100644
--- a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
+++ b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
@@ -26,6 +26,7 @@
<stream-name> name of the Kinesis stream (ie. mySparkStream)
<endpoint-url> endpoint of the Kinesis service
(e.g. https://kinesis.us-east-1.amazonaws.com)
+ <region-name> region name of the Kinesis endpoint (e.g. us-east-1)
Example:
@@ -34,10 +35,10 @@
$ export AWS_SECRET_KEY=<your-secret-key>
# run the example
- $ bin/spark-submit -jars external/kinesis-asl/target/scala-*/\
- spark-streaming-kinesis-asl-assembly_*.jar \
+ $ bin/spark-submit --jars \
+ 'external/kinesis-asl-assembly/target/spark-streaming-kinesis-asl-assembly_*.jar' \
external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
- myAppName mySparkStream https://kinesis.us-east-1.amazonaws.com
+ myAppName mySparkStream https://kinesis.us-east-1.amazonaws.com us-east-1
There is a companion helper class called KinesisWordProducerASL which puts dummy data
onto the Kinesis stream.
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
index fcb790e..d97ab74 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
@@ -78,7 +78,7 @@ object KinesisWordCountASL extends Logging {
if (args.length != 3) {
System.err.println(
"""
- |Usage: KinesisWordCountASL <app-name> <stream-name> <endpoint-url> <region-name>
+ |Usage: KinesisWordCountASL <app-name> <stream-name> <endpoint-url>
|
| <app-name> is the name of the consumer app, used to track the read data in DynamoDB
| <stream-name> is the name of the Kinesis stream
@@ -171,11 +171,11 @@ object KinesisWordCountASL extends Logging {
* <endpoint-url> is the endpoint of the Kinesis service
* (ie. https://kinesis.us-east-1.amazonaws.com)
* <records-per-sec> is the rate of records per second to put onto the stream
- * <words-per-record> is the rate of records per second to put onto the stream
+ * <words-per-record> is the number of words per record
*
* Example:
* $ SPARK_HOME/bin/run-example streaming.KinesisWordProducerASL mySparkStream \
- * https://kinesis.us-east-1.amazonaws.com us-east-1 10 5
+ * https://kinesis.us-east-1.amazonaws.com 10 5
*/
object KinesisWordProducerASL {
def main(args: Array[String]) {
@@ -183,13 +183,13 @@ object KinesisWordProducerASL {
System.err.println(
"""
|Usage: KinesisWordProducerASL <stream-name> <endpoint-url> <records-per-sec>
- <words-per-record>
+ | <words-per-record>
|
| <stream-name> is the name of the Kinesis stream
| <endpoint-url> is the endpoint of the Kinesis service
| (e.g. https://kinesis.us-east-1.amazonaws.com)
| <records-per-sec> is the rate of records per second to put onto the stream
- | <words-per-record> is the rate of records per second to put onto the stream
+ | <words-per-record> is the number of words per record
|
""".stripMargin)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org