You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@phoenix.apache.org by Konstantinos Kougios <ko...@googlemail.com> on 2015/09/28 21:35:03 UTC
WARN client.ScannerCallable: Ignore, probably already closed
I've got a 500 mil rows table on a fairly mediocre cluster. I had some
timeout issues running a count(*) but those are sorted. Now I get a diff
issue, any ideas?
15/09/28 20:24:07 WARN client.ScannerCallable: Ignore, probably already
closed
org.apache.hadoop.hbase.UnknownScannerException:
org.apache.hadoop.hbase.UnknownScannerException: Name: 874, already closed?
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2223)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2114)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
at
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
at java.lang.Thread.run(Thread.java:745)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at
org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
at
org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
at
org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:322)
at
org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:357)
at
org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:195)
at
org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:142)
at
org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:61)
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
at
org.apache.hadoop.hbase.client.StatsTrackingRpcRetryingCaller.callWithoutRetries(StatsTrackingRpcRetryingCaller.java:56)
at
org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320)
at
org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:258)
at
org.apache.hadoop.hbase.client.ClientScanner.possiblyNextScanner(ClientScanner.java:241)
at
org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:532)
at
org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:364)
at
org.apache.phoenix.iterate.ScanningResultIterator.next(ScanningResultIterator.java:55)
at
org.apache.phoenix.iterate.TableResultIterator.next(TableResultIterator.java:107)
at
org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:125)
at
org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:83)
at
org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:62)
at
org.apache.phoenix.iterate.SpoolingResultIterator$SpoolingResultIteratorFactory.newIterator(SpoolingResultIterator.java:78)
at
org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:109)
at
org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:100)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
org.apache.phoenix.job.JobManager$InstrumentedJobFutureTask.run(JobManager.java:183)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by:
org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.UnknownScannerException):
org.apache.hadoop.hbase.UnknownScannerException: Name: 874, already closed?
at
org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2223)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2114)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
at
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
at java.lang.Thread.run(Thread.java:745)
at
org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1196)
at
org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:213)
at
org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:287)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:32651)
at
org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:355)
... 23 more
Re: WARN client.ScannerCallable: Ignore, probably already closed
Posted by Samarth Jain <sa...@gmail.com>.
This likely has to do with hbase scanners running into lease expiration.
Try overriding the value of hbase.client.scanner.timeout.period in the
server side hbase-site.xml to a large value.
We have a feature coming out in Phoenix 4.7 (soon to be released) that will
take care of automatically renewing scanner leases for such long running
queries. To take advantage of that feature, you would need to upgrade your
HBase version to a version as recent as 0.98.17 if you are using HBase-0.98
or certain 1.x versions (1.3.0
<https://issues.apache.org/jira/browse/HBASE/fixforversion/12332794>, 1.2.1
<https://issues.apache.org/jira/browse/HBASE/fixforversion/12332824>, 1.0.3
<https://issues.apache.org/jira/browse/HBASE/fixforversion/12332968>, 1.1.3
<https://issues.apache.org/jira/browse/HBASE/fixforversion/12333152>) if
you are using HBase-1.x.
On Tue, Jan 19, 2016 at 2:13 PM, Nick Dimiduk <nd...@gmail.com> wrote:
> What version of HBase are you using with Phoenix? Does your query ever
> finish -- is it log noise or does it lead to real timeouts?
>
> On Mon, Sep 28, 2015 at 12:35 PM, Konstantinos Kougios <
> kostas.kougios@googlemail.com> wrote:
>
>> I've got a 500 mil rows table on a fairly mediocre cluster. I had some
>> timeout issues running a count(*) but those are sorted. Now I get a diff
>> issue, any ideas?
>>
>> 15/09/28 20:24:07 WARN client.ScannerCallable: Ignore, probably already
>> closed
>> org.apache.hadoop.hbase.UnknownScannerException:
>> org.apache.hadoop.hbase.UnknownScannerException: Name: 874, already closed?
>> at
>> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2223)
>> at
>> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205)
>> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2114)
>> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
>> at
>> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
>> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>> Method)
>> at
>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
>> at
>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>> at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
>> at
>> org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
>> at
>> org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
>> at
>> org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:322)
>> at
>> org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:357)
>> at
>> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:195)
>> at
>> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:142)
>> at
>> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:61)
>> at
>> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
>> at
>> org.apache.hadoop.hbase.client.StatsTrackingRpcRetryingCaller.callWithoutRetries(StatsTrackingRpcRetryingCaller.java:56)
>> at
>> org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320)
>> at
>> org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:258)
>> at
>> org.apache.hadoop.hbase.client.ClientScanner.possiblyNextScanner(ClientScanner.java:241)
>> at
>> org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:532)
>> at
>> org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:364)
>> at
>> org.apache.phoenix.iterate.ScanningResultIterator.next(ScanningResultIterator.java:55)
>> at
>> org.apache.phoenix.iterate.TableResultIterator.next(TableResultIterator.java:107)
>> at
>> org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:125)
>> at
>> org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:83)
>> at
>> org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:62)
>> at
>> org.apache.phoenix.iterate.SpoolingResultIterator$SpoolingResultIteratorFactory.newIterator(SpoolingResultIterator.java:78)
>> at
>> org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:109)
>> at
>> org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:100)
>> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> at
>> org.apache.phoenix.job.JobManager$InstrumentedJobFutureTask.run(JobManager.java:183)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> at java.lang.Thread.run(Thread.java:745)
>> Caused by:
>> org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.UnknownScannerException):
>> org.apache.hadoop.hbase.UnknownScannerException: Name: 874, already closed?
>> at
>> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2223)
>> at
>> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205)
>> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2114)
>> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
>> at
>> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
>> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> at
>> org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1196)
>> at
>> org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:213)
>> at
>> org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:287)
>> at
>> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:32651)
>> at
>> org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:355)
>> ... 23 more
>>
>>
>
Re: WARN client.ScannerCallable: Ignore, probably already closed
Posted by Nick Dimiduk <nd...@gmail.com>.
What version of HBase are you using with Phoenix? Does your query ever
finish -- is it log noise or does it lead to real timeouts?
On Mon, Sep 28, 2015 at 12:35 PM, Konstantinos Kougios <
kostas.kougios@googlemail.com> wrote:
> I've got a 500 mil rows table on a fairly mediocre cluster. I had some
> timeout issues running a count(*) but those are sorted. Now I get a diff
> issue, any ideas?
>
> 15/09/28 20:24:07 WARN client.ScannerCallable: Ignore, probably already
> closed
> org.apache.hadoop.hbase.UnknownScannerException:
> org.apache.hadoop.hbase.UnknownScannerException: Name: 874, already closed?
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2223)
> at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205)
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2114)
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
> at java.lang.Thread.run(Thread.java:745)
>
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
> Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
> at
> org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
> at
> org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
> at
> org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:322)
> at
> org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:357)
> at
> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:195)
> at
> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:142)
> at
> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:61)
> at
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
> at
> org.apache.hadoop.hbase.client.StatsTrackingRpcRetryingCaller.callWithoutRetries(StatsTrackingRpcRetryingCaller.java:56)
> at
> org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320)
> at
> org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:258)
> at
> org.apache.hadoop.hbase.client.ClientScanner.possiblyNextScanner(ClientScanner.java:241)
> at
> org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:532)
> at
> org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:364)
> at
> org.apache.phoenix.iterate.ScanningResultIterator.next(ScanningResultIterator.java:55)
> at
> org.apache.phoenix.iterate.TableResultIterator.next(TableResultIterator.java:107)
> at
> org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:125)
> at
> org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:83)
> at
> org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:62)
> at
> org.apache.phoenix.iterate.SpoolingResultIterator$SpoolingResultIteratorFactory.newIterator(SpoolingResultIterator.java:78)
> at
> org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:109)
> at
> org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:100)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> org.apache.phoenix.job.JobManager$InstrumentedJobFutureTask.run(JobManager.java:183)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by:
> org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.UnknownScannerException):
> org.apache.hadoop.hbase.UnknownScannerException: Name: 874, already closed?
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2223)
> at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205)
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2114)
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
> at java.lang.Thread.run(Thread.java:745)
>
> at
> org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1196)
> at
> org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:213)
> at
> org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:287)
> at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:32651)
> at
> org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:355)
> ... 23 more
>
>