You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "ThimmeGowda (Jira)" <ji...@apache.org> on 2020/06/11 12:46:00 UTC
[jira] [Updated] (SPARK-31969) StreamingJobProgressListener threw
an exception java.util.NoSuchElementException for Long Running Streaming
Job
[ https://issues.apache.org/jira/browse/SPARK-31969?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
ThimmeGowda updated SPARK-31969:
--------------------------------
Attachment: executor_log
driver_log
> StreamingJobProgressListener threw an exception java.util.NoSuchElementException for Long Running Streaming Job
> ---------------------------------------------------------------------------------------------------------------
>
> Key: SPARK-31969
> URL: https://issues.apache.org/jira/browse/SPARK-31969
> Project: Spark
> Issue Type: Bug
> Components: DStreams
> Affects Versions: 2.4.0
> Environment: Kubernetes
> Spark 2.4.0
> Reporter: ThimmeGowda
> Priority: Major
> Attachments: driver_log, executor_log
>
>
> We are running a long running streaming job and Below exception is seen continuosly after sometime. After the jobs starts all of a sudden our Spark streaming application's batch durations start to increase. At around the same time there starts to appear an error log that does not refer to the application code at all. We couldn't find any other significant errors in the driver logs.
> Refrerred ticket : https://issues.apache.org/jira/browse/SPARK-21065 for similar issue, in our case we are not setting anything for spark.streaming.concurrentJobs and default value is taken.
> \{"type":"log", "level":"ERROR", "name":"STREAMING_OTHERS", "time":"2020-06-09T04:31:43.918Z", "timezone":"UTC", "class":"spark-listener-group-appStatus", "method":"streaming.scheduler.StreamingListenerBus.logError(91)", "log":"Listener StreamingJobProgressListener threw an exception\u000Ajava.util.NoSuchElementException: key not found: 1591677100000 ms\u000A\u0009at scala.collection.MapLike$class.default(MapLike.scala:228)\u000A\u0009at scala.collection.AbstractMap.default(Map.scala:59)\u000A\u0009at scala.collection.mutable.HashMap.apply(HashMap.scala:65)\u000A\u0009at org.apache.spark.streaming.ui.StreamingJobProgressListener.onOutputOperationCompleted(StreamingJobProgressListener.scala:134)\u000A\u0009at org.apache.spark.streaming.scheduler.StreamingListenerBus.doPostEvent(StreamingListenerBus.scala:67)\u000A\u0009at org.apache.spark.streaming.scheduler.StreamingListenerBus.doPostEvent(StreamingListenerBus.scala:29)\u000A\u0009at org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:91)\u000A\u0009at org.apache.spark.streaming.scheduler.StreamingListenerBus.postToAll(StreamingListenerBus.scala:29)\u000A\u0009at org.apache.spark.streaming.scheduler.StreamingListenerBus.onOtherEvent(StreamingListenerBus.scala:43)\u000A\u0009at org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:80)\u000A\u0009at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37)\u000A\u0009at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37)\u000A\u0009at org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:91)\u000A\u0009at org.apache.spark.scheduler.AsyncEventQueue.org$apache$spark$scheduler$AsyncEventQueue$$super$postToAll(AsyncEventQueue.scala:92)\u000A\u0009at org.apache.spark.scheduler.AsyncEventQueue$$anonfun$org$apache$spark$scheduler$AsyncEventQueue$$dispatch$1.apply$mcJ$sp(AsyncEventQueue.scala:92)\u000A\u0009at org.apache.spark.scheduler.AsyncEventQueue$$anonfun$org$apache$spark$scheduler$AsyncEventQueue$$dispatch$1.apply(AsyncEventQueue.scala:87)\u000A\u0009at org.apache.spark.scheduler.AsyncEventQueue$$anonfun$org$apache$spark$scheduler$AsyncEventQueue$$dispatch$1.apply(AsyncEventQueue.scala:87)\u000A\u0009at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)\u000A\u0009at org.apache.spark.scheduler.AsyncEventQueue.org$apache$spark$scheduler$AsyncEventQueue$$dispatch(AsyncEventQueue.scala:87)\u000A\u0009at org.apache.spark.scheduler.AsyncEventQueue$$anon$1$$anonfun$run$1.apply$mcV$sp(AsyncEventQueue.scala:83)\u000A\u0009at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1302)\u000A\u0009at org.apache.spark.scheduler.AsyncEventQueue$$anon$1.run(AsyncEventQueue.scala:82)\u000A"}
> java.util.NoSuchElementException: key not found: 1591677100000 ms
> at scala.collection.MapLike$class.default(MapLike.scala:228) ~[scala-library-2.11.12.jar:?]
> at scala.collection.AbstractMap.default(Map.scala:59) ~[scala-library-2.11.12.jar:?]
> at scala.collection.mutable.HashMap.apply(HashMap.scala:65) ~[scala-library-2.11.12.jar:?]
> at org.apache.spark.streaming.ui.StreamingJobProgressListener.onOutputOperationCompleted(StreamingJobProgressListener.scala:134) ~[spark-streaming_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.streaming.scheduler.StreamingListenerBus.doPostEvent(StreamingListenerBus.scala:67) ~[spark-streaming_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.streaming.scheduler.StreamingListenerBus.doPostEvent(StreamingListenerBus.scala:29) [spark-streaming_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:91) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.streaming.scheduler.StreamingListenerBus.postToAll(StreamingListenerBus.scala:29) [spark-streaming_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.streaming.scheduler.StreamingListenerBus.onOtherEvent(StreamingListenerBus.scala:43) [spark-streaming_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:80) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:91) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.scheduler.AsyncEventQueue.org$apache$spark$scheduler$AsyncEventQueue$$super$postToAll(AsyncEventQueue.scala:92) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.scheduler.AsyncEventQueue$$anonfun$org$apache$spark$scheduler$AsyncEventQueue$$dispatch$1.apply$mcJ$sp(AsyncEventQueue.scala:92) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.scheduler.AsyncEventQueue$$anonfun$org$apache$spark$scheduler$AsyncEventQueue$$dispatch$1.apply(AsyncEventQueue.scala:87) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.scheduler.AsyncEventQueue$$anonfun$org$apache$spark$scheduler$AsyncEventQueue$$dispatch$1.apply(AsyncEventQueue.scala:87) [spark-core_2.11-2.4.0.jar:2.4.0]
> at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58) [scala-library-2.11.12.jar:?]
> at org.apache.spark.scheduler.AsyncEventQueue.org$apache$spark$scheduler$AsyncEventQueue$$dispatch(AsyncEventQueue.scala:87) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.scheduler.AsyncEventQueue$$anon$1$$anonfun$run$1.apply$mcV$sp(AsyncEventQueue.scala:83) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1302) [spark-core_2.11-2.4.0.jar:2.4.0]
> at org.apache.spark.scheduler.AsyncEventQueue$$anon$1.run(AsyncEventQueue.scala:82) [spark-core_2.11-2.4.0.jar:2.4.0]
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org