You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2013/12/31 19:13:11 UTC

[08/16] git commit: Minor formatting fixes.

Minor formatting fixes.


Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/0af7f84c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/0af7f84c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/0af7f84c

Branch: refs/heads/master
Commit: 0af7f84c8eb631cd2e427b692f407ec2d37dad64
Parents: 8ca14a1
Author: Tathagata Das <ta...@gmail.com>
Authored: Mon Dec 23 17:47:16 2013 -0800
Committer: Tathagata Das <ta...@gmail.com>
Committed: Mon Dec 23 17:47:16 2013 -0800

----------------------------------------------------------------------
 .../scala/org/apache/spark/streaming/Checkpoint.scala   | 12 ++++++++----
 .../org/apache/spark/streaming/StreamingContext.scala   |  2 --
 .../spark/streaming/dstream/FileInputDStream.scala      |  8 +++++---
 3 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0af7f84c/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala b/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala
index 957c227..0f9a719 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala
@@ -81,7 +81,8 @@ class CheckpointWriter(checkpointDir: String, hadoopConf: Configuration) extends
         attempts += 1
         try {
           logDebug("Saving checkpoint for time " + checkpointTime + " to file '" + file + "'")
-          // This is inherently thread unsafe .. so alleviating it by writing to '.new' and then doing moves : which should be pretty fast.
+          // This is inherently thread unsafe .. so alleviating it by writing to '.new' and
+          // then doing moves : which should be pretty fast.
           val fos = fs.create(writeFile)
           fos.write(bytes)
           fos.close()
@@ -132,7 +133,8 @@ class CheckpointWriter(checkpointDir: String, hadoopConf: Configuration) extends
     val startTime = System.currentTimeMillis()
     val terminated = executor.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS)
     val endTime = System.currentTimeMillis()
-    logInfo("CheckpointWriter executor terminated ? " + terminated + ", waited for " + (endTime - startTime) + " ms.")
+    logInfo("CheckpointWriter executor terminated ? " + terminated +
+      ", waited for " + (endTime - startTime) + " ms.")
   }
 
   private def fs = synchronized {
@@ -151,7 +153,8 @@ object CheckpointReader extends Logging {
 
   def read(path: String): Checkpoint = {
     val fs = new Path(path).getFileSystem(new Configuration())
-    val attempts = Seq(new Path(path, "graph"), new Path(path, "graph.bk"), new Path(path), new Path(path + ".bk"))
+    val attempts = Seq(new Path(path, "graph"), new Path(path, "graph.bk"),
+      new Path(path), new Path(path + ".bk"))
 
     val compressionCodec = CompressionCodec.createCodec()
 
@@ -166,7 +169,8 @@ object CheckpointReader extends Logging {
           // loader to find and load classes. This is a well know Java issue and has popped up
           // in other places (e.g., http://jira.codehaus.org/browse/GROOVY-1627)
           val zis = compressionCodec.compressedInputStream(fis)
-          val ois = new ObjectInputStreamWithLoader(zis, Thread.currentThread().getContextClassLoader)
+          val ois = new ObjectInputStreamWithLoader(zis,
+            Thread.currentThread().getContextClassLoader)
           val cp = ois.readObject.asInstanceOf[Checkpoint]
           ois.close()
           fs.close()

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0af7f84c/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
index 71065f9..6628fdc 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
@@ -50,8 +50,6 @@ import org.apache.hadoop.fs.Path
 import twitter4j.Status
 import twitter4j.auth.Authorization
 
-
-
 /**
  * A StreamingContext is the main entry point for Spark Streaming functionality. Besides the basic
  * information (such as, cluster URL and job name) to internally create a SparkContext, it provides

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0af7f84c/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala
index b526a43..4a7c5cf 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala
@@ -73,7 +73,7 @@ class FileInputDStream[K: ClassTag, V: ClassTag, F <: NewInputFormat[K,V] : Clas
    */
   override def compute(validTime: Time): Option[RDD[(K, V)]] = {
     assert(validTime.milliseconds >= prevModTime,
-      "Trying to get new files for really old time [" + validTime + " < " + prevModTime)
+      "Trying to get new files for really old time [" + validTime + " < " + prevModTime + "]")
 
     // Find new files
     val (newFiles, latestModTime, latestModTimeFiles) = findNewFiles(validTime.milliseconds)
@@ -115,8 +115,8 @@ class FileInputDStream[K: ClassTag, V: ClassTag, F <: NewInputFormat[K,V] : Clas
       attempts += 1
       try {
         val filter = new CustomPathFilter(currentTime)
-        val newFiles = fs.listStatus(path, filter)
-        return (newFiles.map(_.getPath.toString), filter.latestModTime, filter.latestModTimeFiles.toSeq)
+        val newFiles = fs.listStatus(path, filter).map(_.getPath.toString)
+        return (newFiles, filter.latestModTime, filter.latestModTimeFiles.toSeq)
       } catch {
         case ioe: IOException =>
           logWarning("Attempt " + attempts + " to get new files failed", ioe)
@@ -238,10 +238,12 @@ private[streaming]
 object FileInputDStream {
   def defaultFilter(path: Path): Boolean = !path.getName().startsWith(".")
 
+  // Disable slack time (i.e. set it to zero)
   private[streaming] def disableSlackTime() {
     System.setProperty("spark.streaming.fileStream.slackTime", "0")
   }
 
+  // Restore default value of slack time
   private[streaming] def restoreSlackTime() {
     System.clearProperty("spark.streaming.fileStream.slackTime")
   }