You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/08/12 09:01:02 UTC
spark git commit: [MINOR][DOC] Fix style in examples across
documentation
Repository: spark
Updated Branches:
refs/heads/master 993923c8f -> f4482225c
[MINOR][DOC] Fix style in examples across documentation
## What changes were proposed in this pull request?
This PR fixes the documentation as below:
- Python has 4 spaces and Java and Scala has 2 spaces (See https://cwiki.apache.org/confluence/display/SPARK/Spark+Code+Style+Guide).
- Avoid excessive parentheses and curly braces for anonymous functions. (See https://github.com/databricks/scala-style-guide#anonymous)
## How was this patch tested?
N/A
Author: hyukjinkwon <gu...@gmail.com>
Closes #14593 from HyukjinKwon/minor-documentation.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f4482225
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f4482225
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f4482225
Branch: refs/heads/master
Commit: f4482225c405b9cfe078deac74e4c28e2dcc97c3
Parents: 993923c
Author: hyukjinkwon <gu...@gmail.com>
Authored: Fri Aug 12 10:00:58 2016 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Fri Aug 12 10:00:58 2016 +0100
----------------------------------------------------------------------
docs/graphx-programming-guide.md | 8 +++---
docs/programming-guide.md | 4 +--
docs/spark-standalone.md | 6 ++--
docs/streaming-custom-receivers.md | 48 ++++++++++++++++----------------
docs/streaming-programming-guide.md | 28 +++++++++----------
5 files changed, 47 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f4482225/docs/graphx-programming-guide.md
----------------------------------------------------------------------
diff --git a/docs/graphx-programming-guide.md b/docs/graphx-programming-guide.md
index bf4b968..6f738f0 100644
--- a/docs/graphx-programming-guide.md
+++ b/docs/graphx-programming-guide.md
@@ -421,15 +421,15 @@ val graph = Graph(users, relationships, defaultUser)
// Notice that there is a user 0 (for which we have no information) connected to users
// 4 (peter) and 5 (franklin).
graph.triplets.map(
- triplet => triplet.srcAttr._1 + " is the " + triplet.attr + " of " + triplet.dstAttr._1
- ).collect.foreach(println(_))
+ triplet => triplet.srcAttr._1 + " is the " + triplet.attr + " of " + triplet.dstAttr._1
+).collect.foreach(println(_))
// Remove missing vertices as well as the edges to connected to them
val validGraph = graph.subgraph(vpred = (id, attr) => attr._2 != "Missing")
// The valid subgraph will disconnect users 4 and 5 by removing user 0
validGraph.vertices.collect.foreach(println(_))
validGraph.triplets.map(
- triplet => triplet.srcAttr._1 + " is the " + triplet.attr + " of " + triplet.dstAttr._1
- ).collect.foreach(println(_))
+ triplet => triplet.srcAttr._1 + " is the " + triplet.attr + " of " + triplet.dstAttr._1
+).collect.foreach(println(_))
{% endhighlight %}
> Note in the above example only the vertex predicate is provided. The `subgraph` operator defaults
http://git-wip-us.apache.org/repos/asf/spark/blob/f4482225/docs/programming-guide.md
----------------------------------------------------------------------
diff --git a/docs/programming-guide.md b/docs/programming-guide.md
index f828329..40287d7 100644
--- a/docs/programming-guide.md
+++ b/docs/programming-guide.md
@@ -1516,8 +1516,8 @@ data.map(x -> { accum.add(x); return f(x); });
{% highlight python %}
accum = sc.accumulator(0)
def g(x):
- accum.add(x)
- return f(x)
+ accum.add(x)
+ return f(x)
data.map(g)
# Here, accum is still 0 because no actions have caused the `map` to be computed.
{% endhighlight %}
http://git-wip-us.apache.org/repos/asf/spark/blob/f4482225/docs/spark-standalone.md
----------------------------------------------------------------------
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 5ae63fe..1097f1f 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -298,9 +298,9 @@ application at a time. You can cap the number of cores by setting `spark.cores.m
{% highlight scala %}
val conf = new SparkConf()
- .setMaster(...)
- .setAppName(...)
- .set("spark.cores.max", "10")
+ .setMaster(...)
+ .setAppName(...)
+ .set("spark.cores.max", "10")
val sc = new SparkContext(conf)
{% endhighlight %}
http://git-wip-us.apache.org/repos/asf/spark/blob/f4482225/docs/streaming-custom-receivers.md
----------------------------------------------------------------------
diff --git a/docs/streaming-custom-receivers.md b/docs/streaming-custom-receivers.md
index 479140f..fae5901 100644
--- a/docs/streaming-custom-receivers.md
+++ b/docs/streaming-custom-receivers.md
@@ -59,8 +59,8 @@ class CustomReceiver(host: String, port: Int)
}
def onStop() {
- // There is nothing much to do as the thread calling receive()
- // is designed to stop by itself if isStopped() returns false
+ // There is nothing much to do as the thread calling receive()
+ // is designed to stop by itself if isStopped() returns false
}
/** Create a socket connection and receive data until receiver is stopped */
@@ -68,29 +68,29 @@ class CustomReceiver(host: String, port: Int)
var socket: Socket = null
var userInput: String = null
try {
- // Connect to host:port
- socket = new Socket(host, port)
-
- // Until stopped or connection broken continue reading
- val reader = new BufferedReader(
- new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))
- userInput = reader.readLine()
- while(!isStopped && userInput != null) {
- store(userInput)
- userInput = reader.readLine()
- }
- reader.close()
- socket.close()
-
- // Restart in an attempt to connect again when server is active again
- restart("Trying to connect again")
+ // Connect to host:port
+ socket = new Socket(host, port)
+
+ // Until stopped or connection broken continue reading
+ val reader = new BufferedReader(
+ new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))
+ userInput = reader.readLine()
+ while(!isStopped && userInput != null) {
+ store(userInput)
+ userInput = reader.readLine()
+ }
+ reader.close()
+ socket.close()
+
+ // Restart in an attempt to connect again when server is active again
+ restart("Trying to connect again")
} catch {
- case e: java.net.ConnectException =>
- // restart if could not connect to server
- restart("Error connecting to " + host + ":" + port, e)
- case t: Throwable =>
- // restart if there is any other error
- restart("Error receiving data", t)
+ case e: java.net.ConnectException =>
+ // restart if could not connect to server
+ restart("Error connecting to " + host + ":" + port, e)
+ case t: Throwable =>
+ // restart if there is any other error
+ restart("Error receiving data", t)
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f4482225/docs/streaming-programming-guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming-programming-guide.md b/docs/streaming-programming-guide.md
index 3d40b2c..aef62ea 100644
--- a/docs/streaming-programming-guide.md
+++ b/docs/streaming-programming-guide.md
@@ -863,7 +863,7 @@ Java code, take a look at the example
{% highlight python %}
def updateFunction(newValues, runningCount):
if runningCount is None:
- runningCount = 0
+ runningCount = 0
return sum(newValues, runningCount) # add the new values with the previous running count to get the new count
{% endhighlight %}
@@ -903,10 +903,10 @@ spam information (maybe generated with Spark as well) and then filtering based o
{% highlight scala %}
val spamInfoRDD = ssc.sparkContext.newAPIHadoopRDD(...) // RDD containing spam information
-val cleanedDStream = wordCounts.transform(rdd => {
+val cleanedDStream = wordCounts.transform { rdd =>
rdd.join(spamInfoRDD).filter(...) // join data stream with spam information to do data cleaning
...
-})
+}
{% endhighlight %}
</div>
@@ -1142,12 +1142,12 @@ val joinedStream = windowedStream.transform { rdd => rdd.join(dataset) }
JavaPairRDD<String, String> dataset = ...
JavaPairDStream<String, String> windowedStream = stream.window(Durations.seconds(20));
JavaPairDStream<String, String> joinedStream = windowedStream.transform(
- new Function<JavaRDD<Tuple2<String, String>>, JavaRDD<Tuple2<String, String>>>() {
- @Override
- public JavaRDD<Tuple2<String, String>> call(JavaRDD<Tuple2<String, String>> rdd) {
- return rdd.join(dataset);
- }
+ new Function<JavaRDD<Tuple2<String, String>>, JavaRDD<Tuple2<String, String>>>() {
+ @Override
+ public JavaRDD<Tuple2<String, String>> call(JavaRDD<Tuple2<String, String>> rdd) {
+ return rdd.join(dataset);
}
+ }
);
{% endhighlight %}
</div>
@@ -1611,7 +1611,7 @@ words.foreachRDD(
// Do word count on table using SQL and print it
DataFrame wordCountsDataFrame =
- spark.sql("select word, count(*) as total from words group by word");
+ spark.sql("select word, count(*) as total from words group by word");
wordCountsDataFrame.show();
return null;
}
@@ -1759,11 +1759,11 @@ This behavior is made simple by using `StreamingContext.getOrCreate`. This is us
{% highlight scala %}
// Function to create and setup a new StreamingContext
def functionToCreateContext(): StreamingContext = {
- val ssc = new StreamingContext(...) // new context
- val lines = ssc.socketTextStream(...) // create DStreams
- ...
- ssc.checkpoint(checkpointDirectory) // set checkpoint directory
- ssc
+ val ssc = new StreamingContext(...) // new context
+ val lines = ssc.socketTextStream(...) // create DStreams
+ ...
+ ssc.checkpoint(checkpointDirectory) // set checkpoint directory
+ ssc
}
// Get StreamingContext from checkpoint data or create a new one
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org