You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2015/12/28 23:51:28 UTC
spark git commit: [SPARK-12525] Fix fatal compiler warnings in
Kinesis ASL due to @transient annotations
Repository: spark
Updated Branches:
refs/heads/master a6d385322 -> fb572c6e4
[SPARK-12525] Fix fatal compiler warnings in Kinesis ASL due to @transient annotations
The Scala 2.11 SBT build currently fails for Spark 1.6.0 and master due to warnings about the `transient` annotation:
```
[error] [warn] /Users/joshrosen/Documents/spark/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala:73: no valid targets for annotation on value sc - it is discarded unused. You may specify targets with meta-annotations, e.g. (transient param)
[error] [warn] transient sc: SparkContext,
```
This fix implemented here is the same as what we did in #8433: remove the `transient` annotations when they are not necessary and replace use `transient private val` in the remaining cases.
Author: Josh Rosen <jo...@databricks.com>
Closes #10479 from JoshRosen/fix-sbt-2.11.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fb572c6e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fb572c6e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fb572c6e
Branch: refs/heads/master
Commit: fb572c6e4b0645c8084aa013d0c93bb21a79977b
Parents: a6d3853
Author: Josh Rosen <jo...@databricks.com>
Authored: Mon Dec 28 14:51:22 2015 -0800
Committer: Josh Rosen <jo...@databricks.com>
Committed: Mon Dec 28 14:51:22 2015 -0800
----------------------------------------------------------------------
.../streaming/kinesis/KinesisBackedBlockRDD.scala | 14 +++++++-------
.../spark/streaming/kinesis/KinesisInputDStream.scala | 2 +-
2 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fb572c6e/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
----------------------------------------------------------------------
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
index 691c179..3996f16 100644
--- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
+++ b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
@@ -70,26 +70,26 @@ class KinesisBackedBlockRDDPartition(
*/
private[kinesis]
class KinesisBackedBlockRDD[T: ClassTag](
- @transient sc: SparkContext,
+ sc: SparkContext,
val regionName: String,
val endpointUrl: String,
- @transient blockIds: Array[BlockId],
+ @transient private val _blockIds: Array[BlockId],
@transient val arrayOfseqNumberRanges: Array[SequenceNumberRanges],
- @transient isBlockIdValid: Array[Boolean] = Array.empty,
+ @transient private val isBlockIdValid: Array[Boolean] = Array.empty,
val retryTimeoutMs: Int = 10000,
val messageHandler: Record => T = KinesisUtils.defaultMessageHandler _,
val awsCredentialsOption: Option[SerializableAWSCredentials] = None
- ) extends BlockRDD[T](sc, blockIds) {
+ ) extends BlockRDD[T](sc, _blockIds) {
- require(blockIds.length == arrayOfseqNumberRanges.length,
+ require(_blockIds.length == arrayOfseqNumberRanges.length,
"Number of blockIds is not equal to the number of sequence number ranges")
override def isValid(): Boolean = true
override def getPartitions: Array[Partition] = {
- Array.tabulate(blockIds.length) { i =>
+ Array.tabulate(_blockIds.length) { i =>
val isValid = if (isBlockIdValid.length == 0) true else isBlockIdValid(i)
- new KinesisBackedBlockRDDPartition(i, blockIds(i), isValid, arrayOfseqNumberRanges(i))
+ new KinesisBackedBlockRDDPartition(i, _blockIds(i), isValid, arrayOfseqNumberRanges(i))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/fb572c6e/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
----------------------------------------------------------------------
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
index 72ab635..3321c75 100644
--- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
+++ b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
@@ -30,7 +30,7 @@ import org.apache.spark.streaming.scheduler.ReceivedBlockInfo
import org.apache.spark.streaming.{Duration, StreamingContext, Time}
private[kinesis] class KinesisInputDStream[T: ClassTag](
- @transient _ssc: StreamingContext,
+ _ssc: StreamingContext,
streamName: String,
endpointUrl: String,
regionName: String,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org