You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@kudu.apache.org by Darren Hoo <da...@gmail.com> on 2016/03/02 18:35:39 UTC
Spark kudu: scanner not found
when access kudu from Spark SQL, the task throws this error:
16/03/03 01:26:52 WARN client.AsyncKuduScanner:
5d3871ed20a642c28da2f711e3af712f pretends to not know
KuduScanner(table=contents, tablet=5d3871ed20a642c28da2f711e3af712f,
scannerId="2dca9145edf2469789ff851a2db2542a",
scanRequestTimeout=10000)
org.kududb.client.TabletServerErrorException:
Server[8c09eaddf6994d3583b4073447475f8d] NOT_FOUND[code 1]: Scanner
not found
at org.kududb.client.TabletClient.dispatchTSErrorOrReturnException(TabletClient.java:461)
at org.kududb.client.TabletClient.decode(TabletClient.java:412)
at org.kududb.client.TabletClient.decode(TabletClient.java:82)
at org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500)
at org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
at org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at org.kududb.client.TabletClient.handleUpstream(TabletClient.java:592)
at org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.kududb.client.shaded.org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184)
at org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at org.kududb.client.AsyncKuduClient$TabletClientPipeline.sendUpstream(AsyncKuduClient.java:1647)
at org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.kududb.client.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.kududb.client.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
16/03/03 01:26:52 ERROR executor.Executor: Exception in task 1.0 in
stage 3.0 (TID 38)
java.io.IOException: Couldn't get scan data
at org.kududb.mapreduce.KuduTableInputFormat$TableRecordReader.tryRefreshIterator(KuduTableInputFormat.java:422)
at org.kududb.mapreduce.KuduTableInputFormat$TableRecordReader.nextKeyValue(KuduTableInputFormat.java:401)
at org.apache.spark.rdd.NewHadoopRDD$$anon$1.hasNext(NewHadoopRDD.scala:163)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:209)
at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
at org.apache.spark.scheduler.Task.run(Task.scala:88)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.kududb.client.TabletServerErrorException:
Server[8c09eaddf6994d3583b4073447475f8d] NOT_FOUND[code 1]: Scanner
not found
at org.kududb.client.TabletClient.dispatchTSErrorOrReturnException(TabletClient.java:461)
at org.kududb.client.TabletClient.decode(TabletClient.java:412)
at org.kududb.client.TabletClient.decode(TabletClient.java:82)
at org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500)
at org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
at org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at org.kududb.client.TabletClient.handleUpstream(TabletClient.java:592)
at org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.kududb.client.shaded.org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184)
at org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at org.kududb.client.AsyncKuduClient$TabletClientPipeline.sendUpstream(AsyncKuduClient.java:1647)
at org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.kududb.client.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.kududb.client.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
... 3 more
But the tablet server is active:
8c09eaddf6994d3583b4073447475f8d 0.3s rpc_addresses { host:
"slave17" port: 7050 } http_addresses { host: "slave17" port: 8050 }
I could not find any error log from kudu servers.
Version Info:
kudu 0.7.0
revision f9971b3990bb9b687ad7e64a73cc3aedbcc694e7
build type RELEASE
built by jenkins at 29 Feb 2016 00:32:26 PST on
kudu-centos66-1d67.vpc.cloudera.com
build id 2016-02-28_23-47-43
Re: Spark kudu: scanner not found
Posted by Todd Lipcon <to...@cloudera.com>.
Do you see any logs on the tablet server about expiring a scanner? The TS
will close a scanner if it hasn't gotten any requests for a certain amount
of time (1min by default). If the spark job is reading from a scanner very
slowly, this could be the culprit. You could try
setting -scanner_ttl_ms=600000 to get a 10m expiration instead (at the
potential cost of memory if you actually have clients orphaning scanners)
On Wed, Mar 2, 2016 at 5:24 PM, Darren Hoo <da...@gmail.com> wrote:
> hmm, I am already using tip of java client,
> the specific commit is dd6dbda35246722dd28e565252fd4f23c2c8dcad
>
>
> On Thu, Mar 3, 2016 at 4:50 AM, Jean-Daniel Cryans <jd...@apache.org>
> wrote:
> > This looks like: https://issues.apache.org/jira/browse/KUDU-1343
> >
> > We're rolling out a 0.7.1 which will have the fix. It's also really easy
> to
> > patch if you're already building your own java client.
> >
> > Thanks,
> >
> > J-D
> >
> > On Wed, Mar 2, 2016 at 9:35 AM, Darren Hoo <da...@gmail.com> wrote:
> >>
> >> when access kudu from Spark SQL, the task throws this error:
> >>
> >>
> >> 16/03/03 01:26:52 WARN client.AsyncKuduScanner:
> >> 5d3871ed20a642c28da2f711e3af712f pretends to not know
> >> KuduScanner(table=contents, tablet=5d3871ed20a642c28da2f711e3af712f,
> >> scannerId="2dca9145edf2469789ff851a2db2542a",
> >> scanRequestTimeout=10000)
> >> org.kududb.client.TabletServerErrorException:
> >> Server[8c09eaddf6994d3583b4073447475f8d] NOT_FOUND[code 1]: Scanner
> >> not found
> >> at
> >>
> org.kududb.client.TabletClient.dispatchTSErrorOrReturnException(TabletClient.java:461)
> >> at org.kududb.client.TabletClient.decode(TabletClient.java:412)
> >> at org.kududb.client.TabletClient.decode(TabletClient.java:82)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> >> at org.kududb.client.TabletClient.handleUpstream(TabletClient.java:592)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
> >> at
> >>
> org.kududb.client.AsyncKuduClient$TabletClientPipeline.sendUpstream(AsyncKuduClient.java:1647)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> >> at java.lang.Thread.run(Thread.java:745)
> >> 16/03/03 01:26:52 ERROR executor.Executor: Exception in task 1.0 in
> >> stage 3.0 (TID 38)
> >> java.io.IOException: Couldn't get scan data
> >> at
> >>
> org.kududb.mapreduce.KuduTableInputFormat$TableRecordReader.tryRefreshIterator(KuduTableInputFormat.java:422)
> >> at
> >>
> org.kududb.mapreduce.KuduTableInputFormat$TableRecordReader.nextKeyValue(KuduTableInputFormat.java:401)
> >> at
> >>
> org.apache.spark.rdd.NewHadoopRDD$$anon$1.hasNext(NewHadoopRDD.scala:163)
> >> at
> >>
> org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)
> >> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> >> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> >> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> >> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> >> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> >> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> >> at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)
> >> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> >> at
> >>
> org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:209)
> >> at
> >>
> org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73)
> >> at
> >>
> org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
> >> at
> >>
> org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
> >> at org.apache.spark.scheduler.Task.run(Task.scala:88)
> >> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> >> at java.lang.Thread.run(Thread.java:745)
> >> Caused by: org.kududb.client.TabletServerErrorException:
> >> Server[8c09eaddf6994d3583b4073447475f8d] NOT_FOUND[code 1]: Scanner
> >> not found
> >> at
> >>
> org.kududb.client.TabletClient.dispatchTSErrorOrReturnException(TabletClient.java:461)
> >> at org.kududb.client.TabletClient.decode(TabletClient.java:412)
> >> at org.kududb.client.TabletClient.decode(TabletClient.java:82)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> >> at org.kududb.client.TabletClient.handleUpstream(TabletClient.java:592)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
> >> at
> >>
> org.kududb.client.AsyncKuduClient$TabletClientPipeline.sendUpstream(AsyncKuduClient.java:1647)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> >> at
> >>
> org.kududb.client.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> >> ... 3 more
> >>
> >>
> >> But the tablet server is active:
> >>
> >> 8c09eaddf6994d3583b4073447475f8d 0.3s rpc_addresses { host:
> >> "slave17" port: 7050 } http_addresses { host: "slave17" port: 8050 }
> >>
> >>
> >> I could not find any error log from kudu servers.
> >>
> >>
> >> Version Info:
> >>
> >> kudu 0.7.0
> >> revision f9971b3990bb9b687ad7e64a73cc3aedbcc694e7
> >> build type RELEASE
> >> built by jenkins at 29 Feb 2016 00:32:26 PST on
> >> kudu-centos66-1d67.vpc.cloudera.com
> >> build id 2016-02-28_23-47-43
> >
> >
>
--
Todd Lipcon
Software Engineer, Cloudera
Re: Spark kudu: scanner not found
Posted by Darren Hoo <da...@gmail.com>.
hmm, I am already using tip of java client,
the specific commit is dd6dbda35246722dd28e565252fd4f23c2c8dcad
On Thu, Mar 3, 2016 at 4:50 AM, Jean-Daniel Cryans <jd...@apache.org> wrote:
> This looks like: https://issues.apache.org/jira/browse/KUDU-1343
>
> We're rolling out a 0.7.1 which will have the fix. It's also really easy to
> patch if you're already building your own java client.
>
> Thanks,
>
> J-D
>
> On Wed, Mar 2, 2016 at 9:35 AM, Darren Hoo <da...@gmail.com> wrote:
>>
>> when access kudu from Spark SQL, the task throws this error:
>>
>>
>> 16/03/03 01:26:52 WARN client.AsyncKuduScanner:
>> 5d3871ed20a642c28da2f711e3af712f pretends to not know
>> KuduScanner(table=contents, tablet=5d3871ed20a642c28da2f711e3af712f,
>> scannerId="2dca9145edf2469789ff851a2db2542a",
>> scanRequestTimeout=10000)
>> org.kududb.client.TabletServerErrorException:
>> Server[8c09eaddf6994d3583b4073447475f8d] NOT_FOUND[code 1]: Scanner
>> not found
>> at
>> org.kududb.client.TabletClient.dispatchTSErrorOrReturnException(TabletClient.java:461)
>> at org.kududb.client.TabletClient.decode(TabletClient.java:412)
>> at org.kududb.client.TabletClient.decode(TabletClient.java:82)
>> at
>> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500)
>> at
>> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
>> at org.kududb.client.TabletClient.handleUpstream(TabletClient.java:592)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
>> at
>> org.kududb.client.shaded.org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
>> at
>> org.kududb.client.AsyncKuduClient$TabletClientPipeline.sendUpstream(AsyncKuduClient.java:1647)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
>> at
>> org.kududb.client.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
>> at
>> org.kududb.client.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>> at java.lang.Thread.run(Thread.java:745)
>> 16/03/03 01:26:52 ERROR executor.Executor: Exception in task 1.0 in
>> stage 3.0 (TID 38)
>> java.io.IOException: Couldn't get scan data
>> at
>> org.kududb.mapreduce.KuduTableInputFormat$TableRecordReader.tryRefreshIterator(KuduTableInputFormat.java:422)
>> at
>> org.kududb.mapreduce.KuduTableInputFormat$TableRecordReader.nextKeyValue(KuduTableInputFormat.java:401)
>> at
>> org.apache.spark.rdd.NewHadoopRDD$$anon$1.hasNext(NewHadoopRDD.scala:163)
>> at
>> org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)
>> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
>> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
>> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
>> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
>> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
>> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
>> at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)
>> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
>> at
>> org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:209)
>> at
>> org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73)
>> at
>> org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
>> at
>> org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
>> at org.apache.spark.scheduler.Task.run(Task.scala:88)
>> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>> at java.lang.Thread.run(Thread.java:745)
>> Caused by: org.kududb.client.TabletServerErrorException:
>> Server[8c09eaddf6994d3583b4073447475f8d] NOT_FOUND[code 1]: Scanner
>> not found
>> at
>> org.kududb.client.TabletClient.dispatchTSErrorOrReturnException(TabletClient.java:461)
>> at org.kududb.client.TabletClient.decode(TabletClient.java:412)
>> at org.kududb.client.TabletClient.decode(TabletClient.java:82)
>> at
>> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500)
>> at
>> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
>> at org.kududb.client.TabletClient.handleUpstream(TabletClient.java:592)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
>> at
>> org.kududb.client.shaded.org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
>> at
>> org.kududb.client.AsyncKuduClient$TabletClientPipeline.sendUpstream(AsyncKuduClient.java:1647)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
>> at
>> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
>> at
>> org.kududb.client.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
>> at
>> org.kududb.client.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
>> ... 3 more
>>
>>
>> But the tablet server is active:
>>
>> 8c09eaddf6994d3583b4073447475f8d 0.3s rpc_addresses { host:
>> "slave17" port: 7050 } http_addresses { host: "slave17" port: 8050 }
>>
>>
>> I could not find any error log from kudu servers.
>>
>>
>> Version Info:
>>
>> kudu 0.7.0
>> revision f9971b3990bb9b687ad7e64a73cc3aedbcc694e7
>> build type RELEASE
>> built by jenkins at 29 Feb 2016 00:32:26 PST on
>> kudu-centos66-1d67.vpc.cloudera.com
>> build id 2016-02-28_23-47-43
>
>
Re: Spark kudu: scanner not found
Posted by Jean-Daniel Cryans <jd...@apache.org>.
This looks like: https://issues.apache.org/jira/browse/KUDU-1343
We're rolling out a 0.7.1 which will have the fix. It's also really easy to
patch if you're already building your own java client.
Thanks,
J-D
On Wed, Mar 2, 2016 at 9:35 AM, Darren Hoo <da...@gmail.com> wrote:
> when access kudu from Spark SQL, the task throws this error:
>
>
> 16/03/03 01:26:52 WARN client.AsyncKuduScanner:
> 5d3871ed20a642c28da2f711e3af712f pretends to not know
> KuduScanner(table=contents, tablet=5d3871ed20a642c28da2f711e3af712f,
> scannerId="2dca9145edf2469789ff851a2db2542a",
> scanRequestTimeout=10000)
> org.kududb.client.TabletServerErrorException:
> Server[8c09eaddf6994d3583b4073447475f8d] NOT_FOUND[code 1]: Scanner
> not found
> at
> org.kududb.client.TabletClient.dispatchTSErrorOrReturnException(TabletClient.java:461)
> at org.kududb.client.TabletClient.decode(TabletClient.java:412)
> at org.kududb.client.TabletClient.decode(TabletClient.java:82)
> at
> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500)
> at
> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> at org.kududb.client.TabletClient.handleUpstream(TabletClient.java:592)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
> at
> org.kududb.client.shaded.org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
> at
> org.kududb.client.AsyncKuduClient$TabletClientPipeline.sendUpstream(AsyncKuduClient.java:1647)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.kududb.client.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.kududb.client.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> 16/03/03 01:26:52 ERROR executor.Executor: Exception in task 1.0 in
> stage 3.0 (TID 38)
> java.io.IOException: Couldn't get scan data
> at
> org.kududb.mapreduce.KuduTableInputFormat$TableRecordReader.tryRefreshIterator(KuduTableInputFormat.java:422)
> at
> org.kududb.mapreduce.KuduTableInputFormat$TableRecordReader.nextKeyValue(KuduTableInputFormat.java:401)
> at
> org.apache.spark.rdd.NewHadoopRDD$$anon$1.hasNext(NewHadoopRDD.scala:163)
> at
> org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
> at
> org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:209)
> at
> org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73)
> at
> org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
> at
> org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
> at org.apache.spark.scheduler.Task.run(Task.scala:88)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: org.kududb.client.TabletServerErrorException:
> Server[8c09eaddf6994d3583b4073447475f8d] NOT_FOUND[code 1]: Scanner
> not found
> at
> org.kududb.client.TabletClient.dispatchTSErrorOrReturnException(TabletClient.java:461)
> at org.kududb.client.TabletClient.decode(TabletClient.java:412)
> at org.kududb.client.TabletClient.decode(TabletClient.java:82)
> at
> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:500)
> at
> org.kududb.client.shaded.org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> at org.kududb.client.TabletClient.handleUpstream(TabletClient.java:592)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
> at
> org.kududb.client.shaded.org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
> at
> org.kududb.client.AsyncKuduClient$TabletClientPipeline.sendUpstream(AsyncKuduClient.java:1647)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at
> org.kududb.client.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.kududb.client.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.kududb.client.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> ... 3 more
>
>
> But the tablet server is active:
>
> 8c09eaddf6994d3583b4073447475f8d 0.3s rpc_addresses { host:
> "slave17" port: 7050 } http_addresses { host: "slave17" port: 8050 }
>
>
> I could not find any error log from kudu servers.
>
>
> Version Info:
>
> kudu 0.7.0
> revision f9971b3990bb9b687ad7e64a73cc3aedbcc694e7
> build type RELEASE
> built by jenkins at 29 Feb 2016 00:32:26 PST on
> kudu-centos66-1d67.vpc.cloudera.com
> build id 2016-02-28_23-47-43
>