You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@spark.apache.org by Vikas Garg <sp...@gmail.com> on 2021/01/20 12:04:11 UTC
Issue with executer
Hi,
I am facing issue with spark executor. I am struggling with this issue
since last many days and unable to resolve the issue.
Below is the configuration I have given.
val spark = SparkSession.builder()
.appName("Spark Job")
.master("local[*]")
.config("spark.dynamicAllocation.enabled", true)
.config("spark.shuffle.service.enabled", true)
.config("spark.driver.maxResultSize", "8g")
.config("spark.driver.memory", "8g")
.config("spark.executor.memory", "8g")
.config("spark.network.timeout", "3600s")
.getOrCreate()
1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks
*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)
at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)
at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
*RetryingBlockFetcher.java:121*)
at
org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
*NettyBlockTransferService.scala:143*)
at org.apache.spark.network.BlockTransferService.fetchBlockSync(
*BlockTransferService.scala:103*)
at
org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
*BlockManager.scala:1010*)
at
org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
*BlockManager.scala:954*)
at scala.Option.orElse(*Option.scala:289*)
at org.apache.spark.storage.BlockManager.getRemoteBlock(
*BlockManager.scala:954*)
at org.apache.spark.storage.BlockManager.getRemoteBytes(
*BlockManager.scala:1092*)
at
org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
*TaskResultGetter.scala:88*)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
at org.apache.spark.util.Utils$.logUncaughtExceptions(
*Utils.scala:1932*)
at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
*TaskResultGetter.scala:63*)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)
at java.lang.Thread.run(*Thread.java:748*)
Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348
Caused by: *java.net.SocketException*: Permission denied: no further
information
at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)
at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)
at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)
at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks
*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)
at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)
at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
*RetryingBlockFetcher.java:121*)
at
org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
*NettyBlockTransferService.scala:143*)
at org.apache.spark.network.BlockTransferService.fetchBlockSync(
*BlockTransferService.scala:103*)
at
org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
*BlockManager.scala:1010*)
at
org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
*BlockManager.scala:954*)
at scala.Option.orElse(*Option.scala:289*)
at org.apache.spark.storage.BlockManager.getRemoteBlock(
*BlockManager.scala:954*)
at org.apache.spark.storage.BlockManager.getRemoteBytes(
*BlockManager.scala:1092*)
at
org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
*TaskResultGetter.scala:88*)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
at org.apache.spark.util.Utils$.logUncaughtExceptions(
*Utils.scala:1932*)
at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
*TaskResultGetter.scala:63*)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)
at java.lang.Thread.run(*Thread.java:748*)
Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348
Caused by: *java.net.SocketException*: Permission denied: no further
information
at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)
at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)
at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)
at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 1 retries)
*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)
at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)
at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)
at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348
Caused by: *java.net.SocketException*: Permission denied: no further
information
at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)
at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)
at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)
at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 1 retries)
*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)
at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)
at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)
at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348
Caused by: *java.net.SocketException*: Permission denied: no further
information
at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)
at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)
at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)
at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 2 retries)
*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)
at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)
at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)
at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348
Caused by: *java.net.SocketException*: Permission denied: no further
information
at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)
at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)
at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)
at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 2 retries)
*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)
at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)
at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)
at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348
Caused by: *java.net.SocketException*: Permission denied: no further
information
at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)
at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)
at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)
at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 3 retries)
*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)
at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)
at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)
at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348
Caused by: *java.net.SocketException*: Permission denied: no further
information
at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)
at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)
at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)
at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 3 retries)
*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)
at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)
at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)
at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)
at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)
at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348
Caused by: *java.net.SocketException*: Permission denied: no further
information
at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)
at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)
at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)
at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)
at java.lang.Thread.run(*Thread.java:748*)
21/01/20 17:07:12 ERROR TaskSetManager: Task 0 in stage 23.0 failed 1
times; aborting job
21/01/20 17:07:12 ERROR TaskSetManager: Task 2 in stage 17.0 failed 1
times; aborting job
Exception in thread "main"
*org.apache.spark.sql.catalyst.errors.package$TreeNodeException*: execute,
tree:
Exchange hashpartitioning(claimid#1012, 200), true, [id=#742]
at org.apache.spark.sql.catalyst.errors.package$.attachTree(
*package.scala:56*)
at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(
*ShuffleExchangeExec.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)
at org.apache.spark.sql.execution.InputAdapter.inputRDD(
*WholeStageCodegenExec.scala:525*)
at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(
*WholeStageCodegenExec.scala:453*)
at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(
*WholeStageCodegenExec.scala:452*)
at org.apache.spark.sql.execution.InputAdapter.inputRDDs(
*WholeStageCodegenExec.scala:496*)
at org.apache.spark.sql.execution.SortExec.inputRDDs(
*SortExec.scala:132*)
at
org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(
*WholeStageCodegenExec.scala:720*)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)
at
org.apache.spark.sql.execution.joins.SortMergeJoinExec.doExecute(
*SortMergeJoinExec.scala:182*)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)
at org.apache.spark.sql.execution.ProjectExec.doExecute(
*basicPhysicalOperators.scala:76*)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)
at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(
*SparkPlan.scala:316*)
at org.apache.spark.sql.execution.SparkPlan.executeTake(
*SparkPlan.scala:434*)
at org.apache.spark.sql.execution.SparkPlan.executeTake(
*SparkPlan.scala:420*)
at
org.apache.spark.sql.execution.CollectLimitExec.executeCollect(
*limit.scala:47*)
at org.apache.spark.sql.Dataset.collectFromPlan(
*Dataset.scala:3627*)
at org.apache.spark.sql.Dataset.$anonfun$head$1(
*Dataset.scala:2697*)
at org.apache.spark.sql.Dataset.$anonfun$withAction$1(
*Dataset.scala:3618*)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(
*SQLExecution.scala:100*)
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(
*SQLExecution.scala:160*)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(
*SQLExecution.scala:87*)
at org.apache.spark.sql.SparkSession.withActive(
*SparkSession.scala:764*)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(
*SQLExecution.scala:64*)
at org.apache.spark.sql.Dataset.withAction(*Dataset.scala:3616*)
at org.apache.spark.sql.Dataset.head(*Dataset.scala:2697*)
at org.apache.spark.sql.Dataset.take(*Dataset.scala:2904*)
at org.apache.spark.sql.Dataset.getRows(*Dataset.scala:300*)
at org.apache.spark.sql.Dataset.showString(*Dataset.scala:337*)
at org.apache.spark.sql.Dataset.show(*Dataset.scala:824*)
at org.apache.spark.sql.Dataset.show(*Dataset.scala:783*)
at org.apache.spark.sql.Dataset.show(*Dataset.scala:792*)
at com.brighthealthplan.cognizant.output.MedicalClaim$.getDS(
*MedicalClaim.scala:257*)
at
com.brighthealthplan.cognizant.sparkjobs.DataManager.writeLeft(
*DataManager.scala:192*)
at com.brighthealthplan.cognizant.sparkjobs.Main$.main(
*Main.scala:107*)
at
com.brighthealthplan.cognizant.sparkjobs.Main.main(Main.scala)
Caused by: *java.util.concurrent.ExecutionException*:
*org.apache.spark.SparkException*: Job aborted due to stage failure: Task 0
in stage 23.0 failed 1 times, most recent failure: Lost task 0.0 in stage
23.0 (TID 29, del1-lhp-n99999.synapse.com, executor driver): TaskResultLost
(result lost from block manager)
Driver stacktrace:
at java.util.concurrent.FutureTask.report(*FutureTask.java:122*)
at java.util.concurrent.FutureTask.get(*FutureTask.java:206*)
at
org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(
*BroadcastExchangeExec.scala:195*)
at
org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(
*WholeStageCodegenExec.scala:515*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(
*SparkPlan.scala:188*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(
*SparkPlan.scala:184*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(
*BroadcastHashJoinExec.scala:116*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
*BroadcastHashJoinExec.scala:258*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
*BroadcastHashJoinExec.scala:101*)
at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)
at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)
at org.apache.spark.sql.execution.ProjectExec.consume(
*basicPhysicalOperators.scala:41*)
at org.apache.spark.sql.execution.ProjectExec.doConsume(
*basicPhysicalOperators.scala:71*)
at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)
at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(
*BroadcastHashJoinExec.scala:39*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
*BroadcastHashJoinExec.scala:302*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
*BroadcastHashJoinExec.scala:101*)
at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)
at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)
at org.apache.spark.sql.execution.ProjectExec.consume(
*basicPhysicalOperators.scala:41*)
at org.apache.spark.sql.execution.ProjectExec.doConsume(
*basicPhysicalOperators.scala:71*)
at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)
at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(
*BroadcastHashJoinExec.scala:39*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
*BroadcastHashJoinExec.scala:302*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
*BroadcastHashJoinExec.scala:101*)
at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)
at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)
at org.apache.spark.sql.execution.ProjectExec.consume(
*basicPhysicalOperators.scala:41*)
at org.apache.spark.sql.execution.ProjectExec.doConsume(
*basicPhysicalOperators.scala:71*)
at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)
at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)
at org.apache.spark.sql.execution.InputAdapter.consume(
*WholeStageCodegenExec.scala:496*)
at org.apache.spark.sql.execution.InputRDDCodegen.doProduce(
*WholeStageCodegenExec.scala:483*)
at org.apache.spark.sql.execution.InputRDDCodegen.doProduce$(
*WholeStageCodegenExec.scala:456*)
at org.apache.spark.sql.execution.InputAdapter.doProduce(
*WholeStageCodegenExec.scala:496*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.InputAdapter.produce(
*WholeStageCodegenExec.scala:496*)
at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)
at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)
at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)
at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)
at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)
at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)
at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)
at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)
at
org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(
*WholeStageCodegenExec.scala:632*)
at
org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(
*WholeStageCodegenExec.scala:692*)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)
at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)
at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(
*ShuffleExchangeExec.scala:106*)
at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(
*ShuffleExchangeExec.scala:106*)
at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency$lzycompute(
*ShuffleExchangeExec.scala:139*)
at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency(
*ShuffleExchangeExec.scala:137*)
at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.$anonfun$doExecute$1(
*ShuffleExchangeExec.scala:154*)
at org.apache.spark.sql.catalyst.errors.package$.attachTree(
*package.scala:52*)
... 53 more
Caused by: *org.apache.spark.SparkException*: Job aborted due to stage
failure: Task 0 in stage 23.0 failed 1 times, most recent failure: Lost
task 0.0 in stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor
driver): TaskResultLost (result lost from block manager)
Driver stacktrace:
at
org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(
*DAGScheduler.scala:2059*)
at
org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(
*DAGScheduler.scala:2008*)
at
org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(
*DAGScheduler.scala:2007*)
at scala.collection.mutable.ResizableArray.foreach(
*ResizableArray.scala:59*)
at scala.collection.mutable.ResizableArray.foreach$(
*ResizableArray.scala:52*)
at scala.collection.mutable.ArrayBuffer.foreach(
*ArrayBuffer.scala:48*)
at org.apache.spark.scheduler.DAGScheduler.abortStage(
*DAGScheduler.scala:2007*)
at
org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(
*DAGScheduler.scala:973*)
at
org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(
*DAGScheduler.scala:973*)
at scala.Option.foreach(*Option.scala:257*)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(
*DAGScheduler.scala:973*)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(
*DAGScheduler.scala:2239*)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(
*DAGScheduler.scala:2188*)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(
*DAGScheduler.scala:2177*)
at org.apache.spark.util.EventLoop$$anon$1.run(
*EventLoop.scala:49*)
at org.apache.spark.scheduler.DAGScheduler.runJob(
*DAGScheduler.scala:775*)
at org.apache.spark.SparkContext.runJob(
*SparkContext.scala:2099*)
at org.apache.spark.SparkContext.runJob(
*SparkContext.scala:2120*)
at org.apache.spark.SparkContext.runJob(
*SparkContext.scala:2139*)
at org.apache.spark.SparkContext.runJob(
*SparkContext.scala:2164*)
at org.apache.spark.rdd.RDD.$anonfun$collect$1(*RDD.scala:1004*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)
at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:112*)
at org.apache.spark.rdd.RDD.withScope(*RDD.scala:388*)
at org.apache.spark.rdd.RDD.collect(*RDD.scala:1003*)
at
org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(
*SparkPlan.scala:392*)
at
org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(
*BroadcastExchangeExec.scala:120*)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(
*SQLExecution.scala:182*)
at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)
at java.lang.Thread.run(*Thread.java:748*)
Re: Issue with executer
Posted by Vikas Garg <sp...@gmail.com>.
The issue is resolved. Resolution is little weird but it worked. It was due
to scala version mismatch with projects in my package.
On Wed, 20 Jan 2021 at 18:07, Mich Talebzadeh <mi...@gmail.com>
wrote:
> Hi Vikas,
>
> Are you running this on your local laptop etc or using some IDE etc?
>
> What is your available memory for Spark?
>
> Start with minimum set like below
>
> def spark_session_local(appName):
> return SparkSession.builder \
> .master('local[1]') \
> .appName(appName) \
> .enableHiveSupport() \
> .getOrCreate()
>
>
> HTH
>
>
>
> LinkedIn * https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw
> <https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw>*
>
>
>
>
>
> *Disclaimer:* Use it at your own risk. Any and all responsibility for any
> loss, damage or destruction of data or any other property which may arise
> from relying on this email's technical content is explicitly disclaimed.
> The author will in no case be liable for any monetary damages arising from
> such loss, damage or destruction.
>
>
>
>
> On Wed, 20 Jan 2021 at 12:32, Vikas Garg <sp...@gmail.com> wrote:
>
>> Hi Sachit,
>>
>> I am running it in local. My IP mentioned is the private IP address and
>> therefore it is useless for anyone.
>>
>> On Wed, 20 Jan 2021 at 17:37, Sachit Murarka <co...@gmail.com>
>> wrote:
>>
>>> Hi Vikas
>>>
>>> 1. Are you running in local mode? Master has local[*]
>>> 2. Pls mask the ip or confidential info while sharing logs
>>>
>>> Thanks
>>> Sachit
>>>
>>> On Wed, 20 Jan 2021, 17:35 Vikas Garg, <sp...@gmail.com> wrote:
>>>
>>>> Hi,
>>>>
>>>> I am facing issue with spark executor. I am struggling with this issue
>>>> since last many days and unable to resolve the issue.
>>>>
>>>> Below is the configuration I have given.
>>>>
>>>> val spark = SparkSession.builder()
>>>> .appName("Spark Job")
>>>> .master("local[*]")
>>>> .config("spark.dynamicAllocation.enabled", true)
>>>> .config("spark.shuffle.service.enabled", true)
>>>> .config("spark.driver.maxResultSize", "8g")
>>>> .config("spark.driver.memory", "8g")
>>>> .config("spark.executor.memory", "8g")
>>>> .config("spark.network.timeout", "3600s")
>>>> .getOrCreate()
>>>>
>>>> 1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>>>> fetch of 1 outstanding blocks
>>>>
>>>> *java.io.IOException*: Failed to connect to
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:253*)
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:195*)
>>>>
>>>> at
>>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>>> *NettyBlockTransferService.scala:122*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>>> *RetryingBlockFetcher.java:141*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>>>> *RetryingBlockFetcher.java:121*)
>>>>
>>>> at
>>>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>>>> *NettyBlockTransferService.scala:143*)
>>>>
>>>> at
>>>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>>>> *BlockTransferService.scala:103*)
>>>>
>>>> at
>>>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>>>> *BlockManager.scala:1010*)
>>>>
>>>> at
>>>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>>>> *BlockManager.scala:954*)
>>>>
>>>> at scala.Option.orElse(*Option.scala:289*)
>>>>
>>>> at org.apache.spark.storage.BlockManager.getRemoteBlock(
>>>> *BlockManager.scala:954*)
>>>>
>>>> at org.apache.spark.storage.BlockManager.getRemoteBytes(
>>>> *BlockManager.scala:1092*)
>>>>
>>>> at
>>>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>>>> *TaskResultGetter.scala:88*)
>>>>
>>>> at
>>>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>>>
>>>> at org.apache.spark.util.Utils$.logUncaughtExceptions(
>>>> *Utils.scala:1932*)
>>>>
>>>> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>>>> *TaskResultGetter.scala:63*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>>> *ThreadPoolExecutor.java:1149*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>>> *ThreadPoolExecutor.java:624*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>>> Permission denied: no further information:
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>>> information
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native
>>>> Method*)
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>>> *SocketChannelImpl.java:715*)
>>>>
>>>> at
>>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>>> *NioSocketChannel.java:330*)
>>>>
>>>> at
>>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>>> *AbstractNioChannel.java:334*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>>> *NioEventLoop.java:702*)
>>>>
>>>> at
>>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>>> *NioEventLoop.java:650*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>>> *NioEventLoop.java:576*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.run(
>>>> *NioEventLoop.java:493*)
>>>>
>>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>>> *SingleThreadEventExecutor.java:989*)
>>>>
>>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>>> *ThreadExecutorMap.java:74*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> 21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>>>> fetch of 1 outstanding blocks
>>>>
>>>> *java.io.IOException*: Failed to connect to
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:253*)
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:195*)
>>>>
>>>> at
>>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>>> *NettyBlockTransferService.scala:122*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>>> *RetryingBlockFetcher.java:141*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>>>> *RetryingBlockFetcher.java:121*)
>>>>
>>>> at
>>>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>>>> *NettyBlockTransferService.scala:143*)
>>>>
>>>> at
>>>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>>>> *BlockTransferService.scala:103*)
>>>>
>>>> at
>>>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>>>> *BlockManager.scala:1010*)
>>>>
>>>> at
>>>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>>>> *BlockManager.scala:954*)
>>>>
>>>> at scala.Option.orElse(*Option.scala:289*)
>>>>
>>>> at org.apache.spark.storage.BlockManager.getRemoteBlock(
>>>> *BlockManager.scala:954*)
>>>>
>>>> at org.apache.spark.storage.BlockManager.getRemoteBytes(
>>>> *BlockManager.scala:1092*)
>>>>
>>>> at
>>>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>>>> *TaskResultGetter.scala:88*)
>>>>
>>>> at
>>>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>>>
>>>> at org.apache.spark.util.Utils$.logUncaughtExceptions(
>>>> *Utils.scala:1932*)
>>>>
>>>> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>>>> *TaskResultGetter.scala:63*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>>> *ThreadPoolExecutor.java:1149*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>>> *ThreadPoolExecutor.java:624*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>>> Permission denied: no further information:
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>>> information
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native
>>>> Method*)
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>>> *SocketChannelImpl.java:715*)
>>>>
>>>> at
>>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>>> *NioSocketChannel.java:330*)
>>>>
>>>> at
>>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>>> *AbstractNioChannel.java:334*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>>> *NioEventLoop.java:702*)
>>>>
>>>> at
>>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>>> *NioEventLoop.java:650*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>>> *NioEventLoop.java:576*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.run(
>>>> *NioEventLoop.java:493*)
>>>>
>>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>>> *SingleThreadEventExecutor.java:989*)
>>>>
>>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>>> *ThreadExecutorMap.java:74*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>>>> fetch of 1 outstanding blocks (after 1 retries)
>>>>
>>>> *java.io.IOException*: Failed to connect to
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:253*)
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:195*)
>>>>
>>>> at
>>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>>> *NettyBlockTransferService.scala:122*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>>> *RetryingBlockFetcher.java:141*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>>> *RetryingBlockFetcher.java:169*)
>>>>
>>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>>> *Executors.java:511*)
>>>>
>>>> at java.util.concurrent.FutureTask.run(
>>>> *FutureTask.java:266*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>>> *ThreadPoolExecutor.java:1149*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>>> *ThreadPoolExecutor.java:624*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>>> Permission denied: no further information:
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>>> information
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native
>>>> Method*)
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>>> *SocketChannelImpl.java:715*)
>>>>
>>>> at
>>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>>> *NioSocketChannel.java:330*)
>>>>
>>>> at
>>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>>> *AbstractNioChannel.java:334*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>>> *NioEventLoop.java:702*)
>>>>
>>>> at
>>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>>> *NioEventLoop.java:650*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>>> *NioEventLoop.java:576*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.run(
>>>> *NioEventLoop.java:493*)
>>>>
>>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>>> *SingleThreadEventExecutor.java:989*)
>>>>
>>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>>> *ThreadExecutorMap.java:74*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>>>> fetch of 1 outstanding blocks (after 1 retries)
>>>>
>>>> *java.io.IOException*: Failed to connect to
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:253*)
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:195*)
>>>>
>>>> at
>>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>>> *NettyBlockTransferService.scala:122*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>>> *RetryingBlockFetcher.java:141*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>>> *RetryingBlockFetcher.java:169*)
>>>>
>>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>>> *Executors.java:511*)
>>>>
>>>> at java.util.concurrent.FutureTask.run(
>>>> *FutureTask.java:266*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>>> *ThreadPoolExecutor.java:1149*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>>> *ThreadPoolExecutor.java:624*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>>> Permission denied: no further information:
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>>> information
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native
>>>> Method*)
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>>> *SocketChannelImpl.java:715*)
>>>>
>>>> at
>>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>>> *NioSocketChannel.java:330*)
>>>>
>>>> at
>>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>>> *AbstractNioChannel.java:334*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>>> *NioEventLoop.java:702*)
>>>>
>>>> at
>>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>>> *NioEventLoop.java:650*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>>> *NioEventLoop.java:576*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.run(
>>>> *NioEventLoop.java:493*)
>>>>
>>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>>> *SingleThreadEventExecutor.java:989*)
>>>>
>>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>>> *ThreadExecutorMap.java:74*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>>>> fetch of 1 outstanding blocks (after 2 retries)
>>>>
>>>> *java.io.IOException*: Failed to connect to
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:253*)
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:195*)
>>>>
>>>> at
>>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>>> *NettyBlockTransferService.scala:122*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>>> *RetryingBlockFetcher.java:141*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>>> *RetryingBlockFetcher.java:169*)
>>>>
>>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>>> *Executors.java:511*)
>>>>
>>>> at java.util.concurrent.FutureTask.run(
>>>> *FutureTask.java:266*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>>> *ThreadPoolExecutor.java:1149*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>>> *ThreadPoolExecutor.java:624*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>>> Permission denied: no further information:
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>>> information
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native
>>>> Method*)
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>>> *SocketChannelImpl.java:715*)
>>>>
>>>> at
>>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>>> *NioSocketChannel.java:330*)
>>>>
>>>> at
>>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>>> *AbstractNioChannel.java:334*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>>> *NioEventLoop.java:702*)
>>>>
>>>> at
>>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>>> *NioEventLoop.java:650*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>>> *NioEventLoop.java:576*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.run(
>>>> *NioEventLoop.java:493*)
>>>>
>>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>>> *SingleThreadEventExecutor.java:989*)
>>>>
>>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>>> *ThreadExecutorMap.java:74*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>>>> fetch of 1 outstanding blocks (after 2 retries)
>>>>
>>>> *java.io.IOException*: Failed to connect to
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:253*)
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:195*)
>>>>
>>>> at
>>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>>> *NettyBlockTransferService.scala:122*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>>> *RetryingBlockFetcher.java:141*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>>> *RetryingBlockFetcher.java:169*)
>>>>
>>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>>> *Executors.java:511*)
>>>>
>>>> at java.util.concurrent.FutureTask.run(
>>>> *FutureTask.java:266*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>>> *ThreadPoolExecutor.java:1149*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>>> *ThreadPoolExecutor.java:624*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>>> Permission denied: no further information:
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>>> information
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native
>>>> Method*)
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>>> *SocketChannelImpl.java:715*)
>>>>
>>>> at
>>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>>> *NioSocketChannel.java:330*)
>>>>
>>>> at
>>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>>> *AbstractNioChannel.java:334*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>>> *NioEventLoop.java:702*)
>>>>
>>>> at
>>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>>> *NioEventLoop.java:650*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>>> *NioEventLoop.java:576*)
>>>>
>>>> at io.netty.channel.nio.NioEventLoop.run(
>>>> *NioEventLoop.java:493*)
>>>>
>>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>>> *SingleThreadEventExecutor.java:989*)
>>>>
>>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>>> *ThreadExecutorMap.java:74*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> 21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
>>>> fetch of 1 outstanding blocks (after 3 retries)
>>>>
>>>> *java.io.IOException*: Failed to connect to
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:253*)
>>>>
>>>> at
>>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>>> *TransportClientFactory.java:195*)
>>>>
>>>> at
>>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>>> *NettyBlockTransferService.scala:122*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>>> *RetryingBlockFetcher.java:141*)
>>>>
>>>> at
>>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>>> *RetryingBlockFetcher.java:169*)
>>>>
>>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>>> *Executors.java:511*)
>>>>
>>>> at java.util.concurrent.FutureTask.run(
>>>> *FutureTask.java:266*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>>> *ThreadPoolExecutor.java:1149*)
>>>>
>>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>>> *ThreadPoolExecutor.java:624*)
>>>>
>>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>>> *FastThreadLocalRunnable.java:30*)
>>>>
>>>> at java.lang.Thread.run(*Thread.java:748*)
>>>>
>>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>>> Permission denied: no further information:
>>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>>
>>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>>> information
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native
>>>> Method*)
>>>>
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>>> *SocketChannelImpl.java:715*)
>>>>
>>>> at
>>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>>> *NioSocketC*
>>>>
>>>
Re: Issue with executer
Posted by Mich Talebzadeh <mi...@gmail.com>.
Hi Vikas,
Are you running this on your local laptop etc or using some IDE etc?
What is your available memory for Spark?
Start with minimum set like below
def spark_session_local(appName):
return SparkSession.builder \
.master('local[1]') \
.appName(appName) \
.enableHiveSupport() \
.getOrCreate()
HTH
LinkedIn * https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw
<https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw>*
*Disclaimer:* Use it at your own risk. Any and all responsibility for any
loss, damage or destruction of data or any other property which may arise
from relying on this email's technical content is explicitly disclaimed.
The author will in no case be liable for any monetary damages arising from
such loss, damage or destruction.
On Wed, 20 Jan 2021 at 12:32, Vikas Garg <sp...@gmail.com> wrote:
> Hi Sachit,
>
> I am running it in local. My IP mentioned is the private IP address and
> therefore it is useless for anyone.
>
> On Wed, 20 Jan 2021 at 17:37, Sachit Murarka <co...@gmail.com>
> wrote:
>
>> Hi Vikas
>>
>> 1. Are you running in local mode? Master has local[*]
>> 2. Pls mask the ip or confidential info while sharing logs
>>
>> Thanks
>> Sachit
>>
>> On Wed, 20 Jan 2021, 17:35 Vikas Garg, <sp...@gmail.com> wrote:
>>
>>> Hi,
>>>
>>> I am facing issue with spark executor. I am struggling with this issue
>>> since last many days and unable to resolve the issue.
>>>
>>> Below is the configuration I have given.
>>>
>>> val spark = SparkSession.builder()
>>> .appName("Spark Job")
>>> .master("local[*]")
>>> .config("spark.dynamicAllocation.enabled", true)
>>> .config("spark.shuffle.service.enabled", true)
>>> .config("spark.driver.maxResultSize", "8g")
>>> .config("spark.driver.memory", "8g")
>>> .config("spark.executor.memory", "8g")
>>> .config("spark.network.timeout", "3600s")
>>> .getOrCreate()
>>>
>>> 1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>>> fetch of 1 outstanding blocks
>>>
>>> *java.io.IOException*: Failed to connect to
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:253*)
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:195*)
>>>
>>> at
>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>> *NettyBlockTransferService.scala:122*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>> *RetryingBlockFetcher.java:141*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>>> *RetryingBlockFetcher.java:121*)
>>>
>>> at
>>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>>> *NettyBlockTransferService.scala:143*)
>>>
>>> at
>>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>>> *BlockTransferService.scala:103*)
>>>
>>> at
>>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>>> *BlockManager.scala:1010*)
>>>
>>> at
>>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>>> *BlockManager.scala:954*)
>>>
>>> at scala.Option.orElse(*Option.scala:289*)
>>>
>>> at org.apache.spark.storage.BlockManager.getRemoteBlock(
>>> *BlockManager.scala:954*)
>>>
>>> at org.apache.spark.storage.BlockManager.getRemoteBytes(
>>> *BlockManager.scala:1092*)
>>>
>>> at
>>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>>> *TaskResultGetter.scala:88*)
>>>
>>> at
>>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>>
>>> at org.apache.spark.util.Utils$.logUncaughtExceptions(
>>> *Utils.scala:1932*)
>>>
>>> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>>> *TaskResultGetter.scala:63*)
>>>
>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>> *ThreadPoolExecutor.java:1149*)
>>>
>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>> *ThreadPoolExecutor.java:624*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>> Permission denied: no further information:
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>> information
>>>
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*
>>> )
>>>
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>> *SocketChannelImpl.java:715*)
>>>
>>> at
>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>> *NioSocketChannel.java:330*)
>>>
>>> at
>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>> *AbstractNioChannel.java:334*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>> *NioEventLoop.java:702*)
>>>
>>> at
>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>> *NioEventLoop.java:650*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>> *NioEventLoop.java:576*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.run(
>>> *NioEventLoop.java:493*)
>>>
>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>> *SingleThreadEventExecutor.java:989*)
>>>
>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>> *ThreadExecutorMap.java:74*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> 21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>>> fetch of 1 outstanding blocks
>>>
>>> *java.io.IOException*: Failed to connect to
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:253*)
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:195*)
>>>
>>> at
>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>> *NettyBlockTransferService.scala:122*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>> *RetryingBlockFetcher.java:141*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>>> *RetryingBlockFetcher.java:121*)
>>>
>>> at
>>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>>> *NettyBlockTransferService.scala:143*)
>>>
>>> at
>>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>>> *BlockTransferService.scala:103*)
>>>
>>> at
>>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>>> *BlockManager.scala:1010*)
>>>
>>> at
>>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>>> *BlockManager.scala:954*)
>>>
>>> at scala.Option.orElse(*Option.scala:289*)
>>>
>>> at org.apache.spark.storage.BlockManager.getRemoteBlock(
>>> *BlockManager.scala:954*)
>>>
>>> at org.apache.spark.storage.BlockManager.getRemoteBytes(
>>> *BlockManager.scala:1092*)
>>>
>>> at
>>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>>> *TaskResultGetter.scala:88*)
>>>
>>> at
>>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>>
>>> at org.apache.spark.util.Utils$.logUncaughtExceptions(
>>> *Utils.scala:1932*)
>>>
>>> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>>> *TaskResultGetter.scala:63*)
>>>
>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>> *ThreadPoolExecutor.java:1149*)
>>>
>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>> *ThreadPoolExecutor.java:624*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>> Permission denied: no further information:
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>> information
>>>
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*
>>> )
>>>
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>> *SocketChannelImpl.java:715*)
>>>
>>> at
>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>> *NioSocketChannel.java:330*)
>>>
>>> at
>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>> *AbstractNioChannel.java:334*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>> *NioEventLoop.java:702*)
>>>
>>> at
>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>> *NioEventLoop.java:650*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>> *NioEventLoop.java:576*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.run(
>>> *NioEventLoop.java:493*)
>>>
>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>> *SingleThreadEventExecutor.java:989*)
>>>
>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>> *ThreadExecutorMap.java:74*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>>> fetch of 1 outstanding blocks (after 1 retries)
>>>
>>> *java.io.IOException*: Failed to connect to
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:253*)
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:195*)
>>>
>>> at
>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>> *NettyBlockTransferService.scala:122*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>> *RetryingBlockFetcher.java:141*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>> *RetryingBlockFetcher.java:169*)
>>>
>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>> *Executors.java:511*)
>>>
>>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*
>>> )
>>>
>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>> *ThreadPoolExecutor.java:1149*)
>>>
>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>> *ThreadPoolExecutor.java:624*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>> Permission denied: no further information:
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>> information
>>>
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*
>>> )
>>>
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>> *SocketChannelImpl.java:715*)
>>>
>>> at
>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>> *NioSocketChannel.java:330*)
>>>
>>> at
>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>> *AbstractNioChannel.java:334*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>> *NioEventLoop.java:702*)
>>>
>>> at
>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>> *NioEventLoop.java:650*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>> *NioEventLoop.java:576*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.run(
>>> *NioEventLoop.java:493*)
>>>
>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>> *SingleThreadEventExecutor.java:989*)
>>>
>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>> *ThreadExecutorMap.java:74*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>>> fetch of 1 outstanding blocks (after 1 retries)
>>>
>>> *java.io.IOException*: Failed to connect to
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:253*)
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:195*)
>>>
>>> at
>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>> *NettyBlockTransferService.scala:122*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>> *RetryingBlockFetcher.java:141*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>> *RetryingBlockFetcher.java:169*)
>>>
>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>> *Executors.java:511*)
>>>
>>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*
>>> )
>>>
>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>> *ThreadPoolExecutor.java:1149*)
>>>
>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>> *ThreadPoolExecutor.java:624*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>> Permission denied: no further information:
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>> information
>>>
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*
>>> )
>>>
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>> *SocketChannelImpl.java:715*)
>>>
>>> at
>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>> *NioSocketChannel.java:330*)
>>>
>>> at
>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>> *AbstractNioChannel.java:334*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>> *NioEventLoop.java:702*)
>>>
>>> at
>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>> *NioEventLoop.java:650*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>> *NioEventLoop.java:576*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.run(
>>> *NioEventLoop.java:493*)
>>>
>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>> *SingleThreadEventExecutor.java:989*)
>>>
>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>> *ThreadExecutorMap.java:74*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>>> fetch of 1 outstanding blocks (after 2 retries)
>>>
>>> *java.io.IOException*: Failed to connect to
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:253*)
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:195*)
>>>
>>> at
>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>> *NettyBlockTransferService.scala:122*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>> *RetryingBlockFetcher.java:141*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>> *RetryingBlockFetcher.java:169*)
>>>
>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>> *Executors.java:511*)
>>>
>>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*
>>> )
>>>
>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>> *ThreadPoolExecutor.java:1149*)
>>>
>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>> *ThreadPoolExecutor.java:624*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>> Permission denied: no further information:
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>> information
>>>
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*
>>> )
>>>
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>> *SocketChannelImpl.java:715*)
>>>
>>> at
>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>> *NioSocketChannel.java:330*)
>>>
>>> at
>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>> *AbstractNioChannel.java:334*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>> *NioEventLoop.java:702*)
>>>
>>> at
>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>> *NioEventLoop.java:650*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>> *NioEventLoop.java:576*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.run(
>>> *NioEventLoop.java:493*)
>>>
>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>> *SingleThreadEventExecutor.java:989*)
>>>
>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>> *ThreadExecutorMap.java:74*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>>> fetch of 1 outstanding blocks (after 2 retries)
>>>
>>> *java.io.IOException*: Failed to connect to
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:253*)
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:195*)
>>>
>>> at
>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>> *NettyBlockTransferService.scala:122*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>> *RetryingBlockFetcher.java:141*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>> *RetryingBlockFetcher.java:169*)
>>>
>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>> *Executors.java:511*)
>>>
>>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*
>>> )
>>>
>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>> *ThreadPoolExecutor.java:1149*)
>>>
>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>> *ThreadPoolExecutor.java:624*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>> Permission denied: no further information:
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>> information
>>>
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*
>>> )
>>>
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>> *SocketChannelImpl.java:715*)
>>>
>>> at
>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>> *NioSocketChannel.java:330*)
>>>
>>> at
>>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>>> *AbstractNioChannel.java:334*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>>> *NioEventLoop.java:702*)
>>>
>>> at
>>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>>> *NioEventLoop.java:650*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>>> *NioEventLoop.java:576*)
>>>
>>> at io.netty.channel.nio.NioEventLoop.run(
>>> *NioEventLoop.java:493*)
>>>
>>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>>> *SingleThreadEventExecutor.java:989*)
>>>
>>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>>> *ThreadExecutorMap.java:74*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> 21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
>>> fetch of 1 outstanding blocks (after 3 retries)
>>>
>>> *java.io.IOException*: Failed to connect to
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:253*)
>>>
>>> at
>>> org.apache.spark.network.client.TransportClientFactory.createClient(
>>> *TransportClientFactory.java:195*)
>>>
>>> at
>>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>>> *NettyBlockTransferService.scala:122*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>>> *RetryingBlockFetcher.java:141*)
>>>
>>> at
>>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>>> *RetryingBlockFetcher.java:169*)
>>>
>>> at java.util.concurrent.Executors$RunnableAdapter.call(
>>> *Executors.java:511*)
>>>
>>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*
>>> )
>>>
>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>>> *ThreadPoolExecutor.java:1149*)
>>>
>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>>> *ThreadPoolExecutor.java:624*)
>>>
>>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>>> *FastThreadLocalRunnable.java:30*)
>>>
>>> at java.lang.Thread.run(*Thread.java:748*)
>>>
>>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>>> Permission denied: no further information:
>>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>>
>>> Caused by: *java.net.SocketException*: Permission denied: no further
>>> information
>>>
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*
>>> )
>>>
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>>> *SocketChannelImpl.java:715*)
>>>
>>> at
>>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>>> *NioSocketC*
>>>
>>
Re: Issue with executer
Posted by Vikas Garg <sp...@gmail.com>.
Hi Sachit,
I am running it in local. My IP mentioned is the private IP address and
therefore it is useless for anyone.
On Wed, 20 Jan 2021 at 17:37, Sachit Murarka <co...@gmail.com>
wrote:
> Hi Vikas
>
> 1. Are you running in local mode? Master has local[*]
> 2. Pls mask the ip or confidential info while sharing logs
>
> Thanks
> Sachit
>
> On Wed, 20 Jan 2021, 17:35 Vikas Garg, <sp...@gmail.com> wrote:
>
>> Hi,
>>
>> I am facing issue with spark executor. I am struggling with this issue
>> since last many days and unable to resolve the issue.
>>
>> Below is the configuration I have given.
>>
>> val spark = SparkSession.builder()
>> .appName("Spark Job")
>> .master("local[*]")
>> .config("spark.dynamicAllocation.enabled", true)
>> .config("spark.shuffle.service.enabled", true)
>> .config("spark.driver.maxResultSize", "8g")
>> .config("spark.driver.memory", "8g")
>> .config("spark.executor.memory", "8g")
>> .config("spark.network.timeout", "3600s")
>> .getOrCreate()
>>
>> 1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>> *RetryingBlockFetcher.java:121*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>> *NettyBlockTransferService.scala:143*)
>>
>> at
>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>> *BlockTransferService.scala:103*)
>>
>> at
>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>> *BlockManager.scala:1010*)
>>
>> at
>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>> *BlockManager.scala:954*)
>>
>> at scala.Option.orElse(*Option.scala:289*)
>>
>> at org.apache.spark.storage.BlockManager.getRemoteBlock(
>> *BlockManager.scala:954*)
>>
>> at org.apache.spark.storage.BlockManager.getRemoteBytes(
>> *BlockManager.scala:1092*)
>>
>> at
>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>> *TaskResultGetter.scala:88*)
>>
>> at
>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>
>> at org.apache.spark.util.Utils$.logUncaughtExceptions(
>> *Utils.scala:1932*)
>>
>> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>> *TaskResultGetter.scala:63*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>> *RetryingBlockFetcher.java:121*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>> *NettyBlockTransferService.scala:143*)
>>
>> at
>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>> *BlockTransferService.scala:103*)
>>
>> at
>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>> *BlockManager.scala:1010*)
>>
>> at
>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>> *BlockManager.scala:954*)
>>
>> at scala.Option.orElse(*Option.scala:289*)
>>
>> at org.apache.spark.storage.BlockManager.getRemoteBlock(
>> *BlockManager.scala:954*)
>>
>> at org.apache.spark.storage.BlockManager.getRemoteBytes(
>> *BlockManager.scala:1092*)
>>
>> at
>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>> *TaskResultGetter.scala:88*)
>>
>> at
>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>
>> at org.apache.spark.util.Utils$.logUncaughtExceptions(
>> *Utils.scala:1932*)
>>
>> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>> *TaskResultGetter.scala:63*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 1 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 1 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 2 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 2 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 3 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(*NioSocketC*
>>
>
Re: Issue with executer
Posted by Sachit Murarka <co...@gmail.com>.
Hi Vikas
1. Are you running in local mode? Master has local[*]
2. Pls mask the ip or confidential info while sharing logs
Thanks
Sachit
On Wed, 20 Jan 2021, 17:35 Vikas Garg, <sp...@gmail.com> wrote:
> Hi,
>
> I am facing issue with spark executor. I am struggling with this issue
> since last many days and unable to resolve the issue.
>
> Below is the configuration I have given.
>
> val spark = SparkSession.builder()
> .appName("Spark Job")
> .master("local[*]")
> .config("spark.dynamicAllocation.enabled", true)
> .config("spark.shuffle.service.enabled", true)
> .config("spark.driver.maxResultSize", "8g")
> .config("spark.driver.memory", "8g")
> .config("spark.executor.memory", "8g")
> .config("spark.network.timeout", "3600s")
> .getOrCreate()
>
> 1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
> fetch of 1 outstanding blocks
>
> *java.io.IOException*: Failed to connect to
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:253*)
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:195*)
>
> at
> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
> *NettyBlockTransferService.scala:122*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
> *RetryingBlockFetcher.java:141*)
>
> at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
> *RetryingBlockFetcher.java:121*)
>
> at
> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
> *NettyBlockTransferService.scala:143*)
>
> at
> org.apache.spark.network.BlockTransferService.fetchBlockSync(
> *BlockTransferService.scala:103*)
>
> at
> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
> *BlockManager.scala:1010*)
>
> at
> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
> *BlockManager.scala:954*)
>
> at scala.Option.orElse(*Option.scala:289*)
>
> at org.apache.spark.storage.BlockManager.getRemoteBlock(
> *BlockManager.scala:954*)
>
> at org.apache.spark.storage.BlockManager.getRemoteBytes(
> *BlockManager.scala:1092*)
>
> at
> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
> *TaskResultGetter.scala:88*)
>
> at
> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>
> at org.apache.spark.util.Utils$.logUncaughtExceptions(
> *Utils.scala:1932*)
>
> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
> *TaskResultGetter.scala:63*)
>
> at java.util.concurrent.ThreadPoolExecutor.runWorker(
> *ThreadPoolExecutor.java:1149*)
>
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
> *ThreadPoolExecutor.java:624*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
> Permission denied: no further information:
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> Caused by: *java.net.SocketException*: Permission denied: no further
> information
>
> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> *SocketChannelImpl.java:715*)
>
> at
> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
> *NioSocketChannel.java:330*)
>
> at
> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
> *AbstractNioChannel.java:334*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
> *NioEventLoop.java:702*)
>
> at
> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
> *NioEventLoop.java:650*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
> *NioEventLoop.java:576*)
>
> at io.netty.channel.nio.NioEventLoop.run(
> *NioEventLoop.java:493*)
>
> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
> *SingleThreadEventExecutor.java:989*)
>
> at io.netty.util.internal.ThreadExecutorMap$2.run(
> *ThreadExecutorMap.java:74*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> 21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
> fetch of 1 outstanding blocks
>
> *java.io.IOException*: Failed to connect to
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:253*)
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:195*)
>
> at
> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
> *NettyBlockTransferService.scala:122*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
> *RetryingBlockFetcher.java:141*)
>
> at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
> *RetryingBlockFetcher.java:121*)
>
> at
> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
> *NettyBlockTransferService.scala:143*)
>
> at
> org.apache.spark.network.BlockTransferService.fetchBlockSync(
> *BlockTransferService.scala:103*)
>
> at
> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
> *BlockManager.scala:1010*)
>
> at
> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
> *BlockManager.scala:954*)
>
> at scala.Option.orElse(*Option.scala:289*)
>
> at org.apache.spark.storage.BlockManager.getRemoteBlock(
> *BlockManager.scala:954*)
>
> at org.apache.spark.storage.BlockManager.getRemoteBytes(
> *BlockManager.scala:1092*)
>
> at
> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
> *TaskResultGetter.scala:88*)
>
> at
> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>
> at org.apache.spark.util.Utils$.logUncaughtExceptions(
> *Utils.scala:1932*)
>
> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
> *TaskResultGetter.scala:63*)
>
> at java.util.concurrent.ThreadPoolExecutor.runWorker(
> *ThreadPoolExecutor.java:1149*)
>
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
> *ThreadPoolExecutor.java:624*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
> Permission denied: no further information:
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> Caused by: *java.net.SocketException*: Permission denied: no further
> information
>
> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> *SocketChannelImpl.java:715*)
>
> at
> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
> *NioSocketChannel.java:330*)
>
> at
> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
> *AbstractNioChannel.java:334*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
> *NioEventLoop.java:702*)
>
> at
> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
> *NioEventLoop.java:650*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
> *NioEventLoop.java:576*)
>
> at io.netty.channel.nio.NioEventLoop.run(
> *NioEventLoop.java:493*)
>
> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
> *SingleThreadEventExecutor.java:989*)
>
> at io.netty.util.internal.ThreadExecutorMap$2.run(
> *ThreadExecutorMap.java:74*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
> fetch of 1 outstanding blocks (after 1 retries)
>
> *java.io.IOException*: Failed to connect to
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:253*)
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:195*)
>
> at
> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
> *NettyBlockTransferService.scala:122*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
> *RetryingBlockFetcher.java:141*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
> *RetryingBlockFetcher.java:169*)
>
> at java.util.concurrent.Executors$RunnableAdapter.call(
> *Executors.java:511*)
>
> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>
> at java.util.concurrent.ThreadPoolExecutor.runWorker(
> *ThreadPoolExecutor.java:1149*)
>
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
> *ThreadPoolExecutor.java:624*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
> Permission denied: no further information:
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> Caused by: *java.net.SocketException*: Permission denied: no further
> information
>
> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> *SocketChannelImpl.java:715*)
>
> at
> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
> *NioSocketChannel.java:330*)
>
> at
> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
> *AbstractNioChannel.java:334*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
> *NioEventLoop.java:702*)
>
> at
> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
> *NioEventLoop.java:650*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
> *NioEventLoop.java:576*)
>
> at io.netty.channel.nio.NioEventLoop.run(
> *NioEventLoop.java:493*)
>
> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
> *SingleThreadEventExecutor.java:989*)
>
> at io.netty.util.internal.ThreadExecutorMap$2.run(
> *ThreadExecutorMap.java:74*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
> fetch of 1 outstanding blocks (after 1 retries)
>
> *java.io.IOException*: Failed to connect to
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:253*)
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:195*)
>
> at
> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
> *NettyBlockTransferService.scala:122*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
> *RetryingBlockFetcher.java:141*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
> *RetryingBlockFetcher.java:169*)
>
> at java.util.concurrent.Executors$RunnableAdapter.call(
> *Executors.java:511*)
>
> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>
> at java.util.concurrent.ThreadPoolExecutor.runWorker(
> *ThreadPoolExecutor.java:1149*)
>
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
> *ThreadPoolExecutor.java:624*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
> Permission denied: no further information:
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> Caused by: *java.net.SocketException*: Permission denied: no further
> information
>
> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> *SocketChannelImpl.java:715*)
>
> at
> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
> *NioSocketChannel.java:330*)
>
> at
> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
> *AbstractNioChannel.java:334*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
> *NioEventLoop.java:702*)
>
> at
> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
> *NioEventLoop.java:650*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
> *NioEventLoop.java:576*)
>
> at io.netty.channel.nio.NioEventLoop.run(
> *NioEventLoop.java:493*)
>
> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
> *SingleThreadEventExecutor.java:989*)
>
> at io.netty.util.internal.ThreadExecutorMap$2.run(
> *ThreadExecutorMap.java:74*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
> fetch of 1 outstanding blocks (after 2 retries)
>
> *java.io.IOException*: Failed to connect to
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:253*)
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:195*)
>
> at
> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
> *NettyBlockTransferService.scala:122*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
> *RetryingBlockFetcher.java:141*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
> *RetryingBlockFetcher.java:169*)
>
> at java.util.concurrent.Executors$RunnableAdapter.call(
> *Executors.java:511*)
>
> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>
> at java.util.concurrent.ThreadPoolExecutor.runWorker(
> *ThreadPoolExecutor.java:1149*)
>
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
> *ThreadPoolExecutor.java:624*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
> Permission denied: no further information:
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> Caused by: *java.net.SocketException*: Permission denied: no further
> information
>
> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> *SocketChannelImpl.java:715*)
>
> at
> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
> *NioSocketChannel.java:330*)
>
> at
> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
> *AbstractNioChannel.java:334*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
> *NioEventLoop.java:702*)
>
> at
> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
> *NioEventLoop.java:650*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
> *NioEventLoop.java:576*)
>
> at io.netty.channel.nio.NioEventLoop.run(
> *NioEventLoop.java:493*)
>
> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
> *SingleThreadEventExecutor.java:989*)
>
> at io.netty.util.internal.ThreadExecutorMap$2.run(
> *ThreadExecutorMap.java:74*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
> fetch of 1 outstanding blocks (after 2 retries)
>
> *java.io.IOException*: Failed to connect to
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:253*)
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:195*)
>
> at
> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
> *NettyBlockTransferService.scala:122*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
> *RetryingBlockFetcher.java:141*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
> *RetryingBlockFetcher.java:169*)
>
> at java.util.concurrent.Executors$RunnableAdapter.call(
> *Executors.java:511*)
>
> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>
> at java.util.concurrent.ThreadPoolExecutor.runWorker(
> *ThreadPoolExecutor.java:1149*)
>
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
> *ThreadPoolExecutor.java:624*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
> Permission denied: no further information:
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> Caused by: *java.net.SocketException*: Permission denied: no further
> information
>
> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> *SocketChannelImpl.java:715*)
>
> at
> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
> *NioSocketChannel.java:330*)
>
> at
> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
> *AbstractNioChannel.java:334*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
> *NioEventLoop.java:702*)
>
> at
> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
> *NioEventLoop.java:650*)
>
> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
> *NioEventLoop.java:576*)
>
> at io.netty.channel.nio.NioEventLoop.run(
> *NioEventLoop.java:493*)
>
> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
> *SingleThreadEventExecutor.java:989*)
>
> at io.netty.util.internal.ThreadExecutorMap$2.run(
> *ThreadExecutorMap.java:74*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> 21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
> fetch of 1 outstanding blocks (after 3 retries)
>
> *java.io.IOException*: Failed to connect to
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:253*)
>
> at
> org.apache.spark.network.client.TransportClientFactory.createClient(
> *TransportClientFactory.java:195*)
>
> at
> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
> *NettyBlockTransferService.scala:122*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
> *RetryingBlockFetcher.java:141*)
>
> at
> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
> *RetryingBlockFetcher.java:169*)
>
> at java.util.concurrent.Executors$RunnableAdapter.call(
> *Executors.java:511*)
>
> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>
> at java.util.concurrent.ThreadPoolExecutor.runWorker(
> *ThreadPoolExecutor.java:1149*)
>
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
> *ThreadPoolExecutor.java:624*)
>
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
> *FastThreadLocalRunnable.java:30*)
>
> at java.lang.Thread.run(*Thread.java:748*)
>
> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
> Permission denied: no further information:
> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>
> Caused by: *java.net.SocketException*: Permission denied: no further
> information
>
> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> *SocketChannelImpl.java:715*)
>
> at
> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(*NioSocketC*
>
Re: Issue with executer
Posted by Vikas Garg <sp...@gmail.com>.
Hi Sean,
This works fine when I run with very less number of files. Looks like the
issue is with memory or may be something else. I am unable to figure it out.
On Wed, 20 Jan 2021 at 17:58, Sean Owen <sr...@gmail.com> wrote:
> Does not look related to Spark - you have network connectivity problems.
>
> On Wed, Jan 20, 2021 at 6:05 AM Vikas Garg <sp...@gmail.com> wrote:
>
>> Hi,
>>
>> I am facing issue with spark executor. I am struggling with this issue
>> since last many days and unable to resolve the issue.
>>
>> Below is the configuration I have given.
>>
>> val spark = SparkSession.builder()
>> .appName("Spark Job")
>> .master("local[*]")
>> .config("spark.dynamicAllocation.enabled", true)
>> .config("spark.shuffle.service.enabled", true)
>> .config("spark.driver.maxResultSize", "8g")
>> .config("spark.driver.memory", "8g")
>> .config("spark.executor.memory", "8g")
>> .config("spark.network.timeout", "3600s")
>> .getOrCreate()
>>
>> 1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>> *RetryingBlockFetcher.java:121*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>> *NettyBlockTransferService.scala:143*)
>>
>> at
>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>> *BlockTransferService.scala:103*)
>>
>> at
>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>> *BlockManager.scala:1010*)
>>
>> at
>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>> *BlockManager.scala:954*)
>>
>> at scala.Option.orElse(*Option.scala:289*)
>>
>> at org.apache.spark.storage.BlockManager.getRemoteBlock(
>> *BlockManager.scala:954*)
>>
>> at org.apache.spark.storage.BlockManager.getRemoteBytes(
>> *BlockManager.scala:1092*)
>>
>> at
>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>> *TaskResultGetter.scala:88*)
>>
>> at
>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>
>> at org.apache.spark.util.Utils$.logUncaughtExceptions(
>> *Utils.scala:1932*)
>>
>> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>> *TaskResultGetter.scala:63*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>> *RetryingBlockFetcher.java:121*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>> *NettyBlockTransferService.scala:143*)
>>
>> at
>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>> *BlockTransferService.scala:103*)
>>
>> at
>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>> *BlockManager.scala:1010*)
>>
>> at
>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>> *BlockManager.scala:954*)
>>
>> at scala.Option.orElse(*Option.scala:289*)
>>
>> at org.apache.spark.storage.BlockManager.getRemoteBlock(
>> *BlockManager.scala:954*)
>>
>> at org.apache.spark.storage.BlockManager.getRemoteBytes(
>> *BlockManager.scala:1092*)
>>
>> at
>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>> *TaskResultGetter.scala:88*)
>>
>> at
>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>
>> at org.apache.spark.util.Utils$.logUncaughtExceptions(
>> *Utils.scala:1932*)
>>
>> at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>> *TaskResultGetter.scala:63*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 1 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 1 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 2 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 2 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 3 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 3 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>> at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>> at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>> at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>> at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>> at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>> at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>> at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>> at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>> at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>> at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>> at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>> at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>> at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:12 ERROR TaskSetManager: Task 0 in stage 23.0 failed 1
>> times; aborting job
>>
>> 21/01/20 17:07:12 ERROR TaskSetManager: Task 2 in stage 17.0 failed 1
>> times; aborting job
>>
>> Exception in thread "main"
>> *org.apache.spark.sql.catalyst.errors.package$TreeNodeException*:
>> execute, tree:
>>
>> Exchange hashpartitioning(claimid#1012, 200), true, [id=#742]
>>
>>
>>
>>
>>
>> at org.apache.spark.sql.catalyst.errors.package$.attachTree(
>> *package.scala:56*)
>>
>> at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(
>> *ShuffleExchangeExec.scala:151*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>> at org.apache.spark.sql.execution.InputAdapter.inputRDD(
>> *WholeStageCodegenExec.scala:525*)
>>
>> at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(
>> *WholeStageCodegenExec.scala:453*)
>>
>> at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(
>> *WholeStageCodegenExec.scala:452*)
>>
>> at org.apache.spark.sql.execution.InputAdapter.inputRDDs(
>> *WholeStageCodegenExec.scala:496*)
>>
>> at org.apache.spark.sql.execution.SortExec.inputRDDs(
>> *SortExec.scala:132*)
>>
>> at
>> org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(
>> *WholeStageCodegenExec.scala:720*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>> at
>> org.apache.spark.sql.execution.joins.SortMergeJoinExec.doExecute(
>> *SortMergeJoinExec.scala:182*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doExecute(
>> *basicPhysicalOperators.scala:76*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(
>> *SparkPlan.scala:316*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeTake(
>> *SparkPlan.scala:434*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeTake(
>> *SparkPlan.scala:420*)
>>
>> at
>> org.apache.spark.sql.execution.CollectLimitExec.executeCollect(
>> *limit.scala:47*)
>>
>> at org.apache.spark.sql.Dataset.collectFromPlan(
>> *Dataset.scala:3627*)
>>
>> at org.apache.spark.sql.Dataset.$anonfun$head$1(
>> *Dataset.scala:2697*)
>>
>> at org.apache.spark.sql.Dataset.$anonfun$withAction$1(
>> *Dataset.scala:3618*)
>>
>> at
>> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(
>> *SQLExecution.scala:100*)
>>
>> at
>> org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(
>> *SQLExecution.scala:160*)
>>
>> at
>> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(
>> *SQLExecution.scala:87*)
>>
>> at org.apache.spark.sql.SparkSession.withActive(
>> *SparkSession.scala:764*)
>>
>> at
>> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(
>> *SQLExecution.scala:64*)
>>
>> at org.apache.spark.sql.Dataset.withAction(
>> *Dataset.scala:3616*)
>>
>> at org.apache.spark.sql.Dataset.head(*Dataset.scala:2697*)
>>
>> at org.apache.spark.sql.Dataset.take(*Dataset.scala:2904*)
>>
>> at org.apache.spark.sql.Dataset.getRows(*Dataset.scala:300*)
>>
>> at org.apache.spark.sql.Dataset.showString(
>> *Dataset.scala:337*)
>>
>> at org.apache.spark.sql.Dataset.show(*Dataset.scala:824*)
>>
>> at org.apache.spark.sql.Dataset.show(*Dataset.scala:783*)
>>
>> at org.apache.spark.sql.Dataset.show(*Dataset.scala:792*)
>>
>> at com.brighthealthplan.cognizant.output.MedicalClaim$.getDS(
>> *MedicalClaim.scala:257*)
>>
>> at
>> com.brighthealthplan.cognizant.sparkjobs.DataManager.writeLeft(
>> *DataManager.scala:192*)
>>
>> at com.brighthealthplan.cognizant.sparkjobs.Main$.main(
>> *Main.scala:107*)
>>
>> at
>> com.brighthealthplan.cognizant.sparkjobs.Main.main(Main.scala)
>>
>> Caused by: *java.util.concurrent.ExecutionException*:
>> *org.apache.spark.SparkException*: Job aborted due to stage failure:
>> Task 0 in stage 23.0 failed 1 times, most recent failure: Lost task 0.0 in
>> stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor driver):
>> TaskResultLost (result lost from block manager)
>>
>> Driver stacktrace:
>>
>> at java.util.concurrent.FutureTask.report(
>> *FutureTask.java:122*)
>>
>> at java.util.concurrent.FutureTask.get(*FutureTask.java:206*)
>>
>> at
>> org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(
>> *BroadcastExchangeExec.scala:195*)
>>
>> at
>> org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(
>> *WholeStageCodegenExec.scala:515*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(
>> *SparkPlan.scala:188*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(
>> *SparkPlan.scala:184*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(
>> *BroadcastHashJoinExec.scala:116*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
>> *BroadcastHashJoinExec.scala:258*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
>> *BroadcastHashJoinExec.scala:101*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.consume(
>> *basicPhysicalOperators.scala:41*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doConsume(
>> *basicPhysicalOperators.scala:71*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(
>> *BroadcastHashJoinExec.scala:39*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
>> *BroadcastHashJoinExec.scala:302*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
>> *BroadcastHashJoinExec.scala:101*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.consume(
>> *basicPhysicalOperators.scala:41*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doConsume(
>> *basicPhysicalOperators.scala:71*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(
>> *BroadcastHashJoinExec.scala:39*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
>> *BroadcastHashJoinExec.scala:302*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
>> *BroadcastHashJoinExec.scala:101*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.consume(
>> *basicPhysicalOperators.scala:41*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doConsume(
>> *basicPhysicalOperators.scala:71*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>> at org.apache.spark.sql.execution.InputAdapter.consume(
>> *WholeStageCodegenExec.scala:496*)
>>
>> at org.apache.spark.sql.execution.InputRDDCodegen.doProduce(
>> *WholeStageCodegenExec.scala:483*)
>>
>> at org.apache.spark.sql.execution.InputRDDCodegen.doProduce$(
>> *WholeStageCodegenExec.scala:456*)
>>
>> at org.apache.spark.sql.execution.InputAdapter.doProduce(
>> *WholeStageCodegenExec.scala:496*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.InputAdapter.produce(
>> *WholeStageCodegenExec.scala:496*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>> at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>> at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>> at
>> org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(
>> *WholeStageCodegenExec.scala:632*)
>>
>> at
>> org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(
>> *WholeStageCodegenExec.scala:692*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>> at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>> at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(
>> *ShuffleExchangeExec.scala:106*)
>>
>> at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(
>> *ShuffleExchangeExec.scala:106*)
>>
>> at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency$lzycompute(
>> *ShuffleExchangeExec.scala:139*)
>>
>> at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency(
>> *ShuffleExchangeExec.scala:137*)
>>
>> at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.$anonfun$doExecute$1(
>> *ShuffleExchangeExec.scala:154*)
>>
>> at org.apache.spark.sql.catalyst.errors.package$.attachTree(
>> *package.scala:52*)
>>
>> ... 53 more
>>
>> Caused by: *org.apache.spark.SparkException*: Job aborted due to stage
>> failure: Task 0 in stage 23.0 failed 1 times, most recent failure: Lost
>> task 0.0 in stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor
>> driver): TaskResultLost (result lost from block manager)
>>
>> Driver stacktrace:
>>
>> at
>> org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(
>> *DAGScheduler.scala:2059*)
>>
>> at
>> org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(
>> *DAGScheduler.scala:2008*)
>>
>> at
>> org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(
>> *DAGScheduler.scala:2007*)
>>
>> at scala.collection.mutable.ResizableArray.foreach(
>> *ResizableArray.scala:59*)
>>
>> at scala.collection.mutable.ResizableArray.foreach$(
>> *ResizableArray.scala:52*)
>>
>> at scala.collection.mutable.ArrayBuffer.foreach(
>> *ArrayBuffer.scala:48*)
>>
>> at org.apache.spark.scheduler.DAGScheduler.abortStage(
>> *DAGScheduler.scala:2007*)
>>
>> at
>> org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(
>> *DAGScheduler.scala:973*)
>>
>> at
>> org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(
>> *DAGScheduler.scala:973*)
>>
>> at scala.Option.foreach(*Option.scala:257*)
>>
>> at
>> org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(
>> *DAGScheduler.scala:973*)
>>
>> at
>> org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(
>> *DAGScheduler.scala:2239*)
>>
>> at
>> org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(
>> *DAGScheduler.scala:2188*)
>>
>> at
>> org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(
>> *DAGScheduler.scala:2177*)
>>
>> at org.apache.spark.util.EventLoop$$anon$1.run(
>> *EventLoop.scala:49*)
>>
>> at org.apache.spark.scheduler.DAGScheduler.runJob(
>> *DAGScheduler.scala:775*)
>>
>> at org.apache.spark.SparkContext.runJob(
>> *SparkContext.scala:2099*)
>>
>> at org.apache.spark.SparkContext.runJob(
>> *SparkContext.scala:2120*)
>>
>> at org.apache.spark.SparkContext.runJob(
>> *SparkContext.scala:2139*)
>>
>> at org.apache.spark.SparkContext.runJob(
>> *SparkContext.scala:2164*)
>>
>> at org.apache.spark.rdd.RDD.$anonfun$collect$1(
>> *RDD.scala:1004*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>> at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:112*)
>>
>> at org.apache.spark.rdd.RDD.withScope(*RDD.scala:388*)
>>
>> at org.apache.spark.rdd.RDD.collect(*RDD.scala:1003*)
>>
>> at
>> org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(
>> *SparkPlan.scala:392*)
>>
>> at
>> org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(
>> *BroadcastExchangeExec.scala:120*)
>>
>> at
>> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(
>> *SQLExecution.scala:182*)
>>
>> at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>> at java.lang.Thread.run(*Thread.java:748*)
>>
>>