You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ko...@apache.org on 2018/08/04 19:44:21 UTC
spark git commit: [SPARK-24987][SS] - Fix Kafka consumer leak when no
new offsets for TopicPartition
Repository: spark
Updated Branches:
refs/heads/master 55e3ae693 -> b7fdf8eb2
[SPARK-24987][SS] - Fix Kafka consumer leak when no new offsets for TopicPartition
## What changes were proposed in this pull request?
This small fix adds a `consumer.release()` call to `KafkaSourceRDD` in the case where we've retrieved offsets from Kafka, but the `fromOffset` is equal to the `lastOffset`, meaning there is no new data to read for a particular topic partition. Up until now, we'd just return an empty iterator without closing the consumer which would cause a FD leak.
If accepted, this pull request should be merged into master as well.
## How was this patch tested?
Haven't ran any specific tests, would love help on how to test methods running inside `RDD.compute`.
Author: Yuval Itzchakov <yu...@clicktale.com>
Closes #21997 from YuvalItzchakov/master.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b7fdf8eb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b7fdf8eb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b7fdf8eb
Branch: refs/heads/master
Commit: b7fdf8eb2011ae76f0161caa9da91e29f52f05e4
Parents: 55e3ae6
Author: Yuval Itzchakov <yu...@clicktale.com>
Authored: Sat Aug 4 14:44:10 2018 -0500
Committer: cody koeninger <co...@koeninger.org>
Committed: Sat Aug 4 14:44:10 2018 -0500
----------------------------------------------------------------------
.../main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b7fdf8eb/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
index 53bd9a9..8b4494d 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
@@ -124,8 +124,6 @@ private[kafka010] class KafkaSourceRDD(
thePart: Partition,
context: TaskContext): Iterator[ConsumerRecord[Array[Byte], Array[Byte]]] = {
val sourcePartition = thePart.asInstanceOf[KafkaSourceRDDPartition]
- val topic = sourcePartition.offsetRange.topic
- val kafkaPartition = sourcePartition.offsetRange.partition
val consumer = KafkaDataConsumer.acquire(
sourcePartition.offsetRange.topicPartition, executorKafkaParams, reuseKafkaConsumer)
@@ -138,6 +136,7 @@ private[kafka010] class KafkaSourceRDD(
if (range.fromOffset == range.untilOffset) {
logInfo(s"Beginning offset ${range.fromOffset} is the same as ending offset " +
s"skipping ${range.topic} ${range.partition}")
+ consumer.release()
Iterator.empty
} else {
val underlying = new NextIterator[ConsumerRecord[Array[Byte], Array[Byte]]]() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org