You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by Jeff Whiting <je...@qualtrics.com> on 2012/07/24 20:52:38 UTC
MR hbase export is failing
What would cause a scanner timeout exception? Is hdfs too slow? Do I just increase the scanner
timeout or is there a better approach.
Thanks,
~Jeff
running: hadoop jar /usr/lib/hbase/hbase-0.90.1-CDH3B4.jar export -D dfs.replication=2 -D
mapred.output.compress=true -D
mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec -D
mapred.output.compression.type=BLOCK response /hbaseBackup/
partial/2012-07-20_2012-07-21/response 1 1342742400000 1342828800000
org.apache.hadoop.hbase.client.ScannerTimeoutException: 61294ms passed since the last invocation,
timeout is currently set to 60000
at org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1114)
at
org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl.nextKeyValue(TableRecordReaderImpl.java:143)
at org.apache.hadoop.hbase.mapreduce.TableRecordReader.nextKeyValue(TableRecordReader.java:142)
at org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.nextKeyValue(MapTask.java:455)
at org.apache.hadoop.mapreduce.MapContext.nextKeyValue(MapContext.java:67)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:646)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:322)
at org.apache.hadoop.mapred.Child$4.run(Child.java:240)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
at org.apache.hadoop.mapred.Child.main(Child.java:234)
Caused by: org.apache.hadoop.hbase.UnknownScannerException:
org.apache.hadoop.hbase.UnknownScannerException: Name: -3564751891935236449
at org.apache.hadoop.hbase.regionserver.HRegionServer.next(HRegionServer.java:1795)
at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at
org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(RemoteExceptionHandler.java:96)
at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:83)
at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:38)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getRegionServerWithRetries(HConnectionManager.java:1000)
at org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1100)
... 12 more
--
Jeff Whiting
Qualtrics Senior Software Engineer
jeffw@qualtrics.com
Re: MR hbase export is failing
Posted by Jimmy Xiang <jx...@cloudera.com>.
It could be also caused by the MR takes too long to process a batch of
data before coming back for another batch.
Thanks,
Jimmy
On Tue, Jul 24, 2012 at 11:52 AM, Jeff Whiting <je...@qualtrics.com> wrote:
> What would cause a scanner timeout exception? Is hdfs too slow? Do I just
> increase the scanner timeout or is there a better approach.
>
> Thanks,
> ~Jeff
>
> running: hadoop jar /usr/lib/hbase/hbase-0.90.1-CDH3B4.jar export -D
> dfs.replication=2 -D mapred.output.compress=true -D
> mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec -D
> mapred.output.compression.type=BLOCK response /hbaseBackup/
> partial/2012-07-20_2012-07-21/response 1 1342742400000 1342828800000
> org.apache.hadoop.hbase.client.ScannerTimeoutException: 61294ms passed since
> the last invocation, timeout is currently set to 60000
> at
> org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1114)
> at
> org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl.nextKeyValue(TableRecordReaderImpl.java:143)
> at
> org.apache.hadoop.hbase.mapreduce.TableRecordReader.nextKeyValue(TableRecordReader.java:142)
> at
> org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.nextKeyValue(MapTask.java:455)
> at
> org.apache.hadoop.mapreduce.MapContext.nextKeyValue(MapContext.java:67)
> at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143)
> at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:646)
> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:322)
> at org.apache.hadoop.mapred.Child$4.run(Child.java:240)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
> at org.apache.hadoop.mapred.Child.main(Child.java:234)
> Caused by: org.apache.hadoop.hbase.UnknownScannerException:
> org.apache.hadoop.hbase.UnknownScannerException: Name: -3564751891935236449
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.next(HRegionServer.java:1795)
> at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
>
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
> Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
> at
> org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(RemoteExceptionHandler.java:96)
> at
> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:83)
> at
> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:38)
> at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getRegionServerWithRetries(HConnectionManager.java:1000)
> at
> org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1100)
> ... 12 more
>
> --
> Jeff Whiting
> Qualtrics Senior Software Engineer
> jeffw@qualtrics.com
>
Re: MR hbase export is failing
Posted by Jeff Whiting <je...@qualtrics.com>.
Thanks for the replies everyone. We are on an old version of hbase and plan on upgrading pretty soon.
Immediately it looks like we have to tell the region server to have a longer timeout rather than
telling the client to have a longer timeout. I was hoping to just change a parameter in the MR job.
We'll run the backup and watch how busy the data node. Typically they seem pretty unloaded. I was
surprised that we were getting timeouts. Our rows aren't really large (the biggest would 1 mb but
typically it is more like 5-10 kb).
~Jeff
On 7/24/2012 8:00 PM, Ooh Rong wrote:
> We had a similar issue on CDH3u4(hbase 0.90) and upgrading to
> CDH4(hbase 0.92) solved the problem for us.
> If it's an option, I recommend upgrading HBase.
>
>
> On Wed, Jul 25, 2012 at 4:04 AM, Paul Mackles <pm...@adobe.com> wrote:
>> I've seen this when writing when exporting to s3 and assumed it was
>> related to write performance. We set hbase.regionserver.lease.period to
>> the same values as the task timeout and it helped reduce the # of failures
>> though we still get occasional task timeouts. I haven't seen this when
>> writing to local HDFS though. Are any of your rows particularly wide? How
>> loaded are your data nodes when running the export? Are you running too
>> many mappers per node and starving hbase?
>>
>> On 7/24/12 2:52 PM, "Jeff Whiting" <je...@qualtrics.com> wrote:
>>
>>> What would cause a scanner timeout exception? Is hdfs too slow? Do I
>>> just increase the scanner
>>> timeout or is there a better approach.
>>>
>>> Thanks,
>>> ~Jeff
>>>
>>> running: hadoop jar /usr/lib/hbase/hbase-0.90.1-CDH3B4.jar export -D
>>> dfs.replication=2 -D
>>> mapred.output.compress=true -D
>>> mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec
>>> -D
>>> mapred.output.compression.type=BLOCK response /hbaseBackup/
>>> partial/2012-07-20_2012-07-21/response 1 1342742400000 1342828800000
>>> org.apache.hadoop.hbase.client.ScannerTimeoutException: 61294ms passed
>>> since the last invocation,
>>> timeout is currently set to 60000
>>> at
>>> org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1114)
>>> at
>>> org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl.nextKeyValue(Table
>>> RecordReaderImpl.java:143)
>>> at
>>> org.apache.hadoop.hbase.mapreduce.TableRecordReader.nextKeyValue(TableReco
>>> rdReader.java:142)
>>> at
>>> org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.nextKeyValue(MapT
>>> ask.java:455)
>>> at
>>> org.apache.hadoop.mapreduce.MapContext.nextKeyValue(MapContext.java:67)
>>> at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143)
>>> at
>>> org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:646)
>>> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:322)
>>> at org.apache.hadoop.mapred.Child$4.run(Child.java:240)
>>> at java.security.AccessController.doPrivileged(Native Method)
>>> at javax.security.auth.Subject.doAs(Subject.java:396)
>>> at
>>> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.
>>> java:1115)
>>> at org.apache.hadoop.mapred.Child.main(Child.java:234)
>>> Caused by: org.apache.hadoop.hbase.UnknownScannerException:
>>> org.apache.hadoop.hbase.UnknownScannerException: Name:
>>> -3564751891935236449
>>> at
>>> org.apache.hadoop.hbase.regionserver.HRegionServer.next(HRegionServer.java
>>> :1795)
>>> at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
>>> at
>>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorIm
>>> pl.java:25)
>>> at java.lang.reflect.Method.invoke(Method.java:597)
>>> at
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
>>> at
>>> org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
>>>
>>> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>> Method)
>>> at
>>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAcc
>>> essorImpl.java:39)
>>> at
>>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstr
>>> uctorAccessorImpl.java:27)
>>> at
>>> java.lang.reflect.Constructor.newInstance(Constructor.java:513)
>>> at
>>> org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(Remot
>>> eExceptionHandler.java:96)
>>> at
>>> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:8
>>> 3)
>>> at
>>> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:3
>>> 8)
>>> at
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementatio
>>> n.getRegionServerWithRetries(HConnectionManager.java:1000)
>>> at
>>> org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1100)
>>> ... 12 more
>>>
>>> --
>>> Jeff Whiting
>>> Qualtrics Senior Software Engineer
>>> jeffw@qualtrics.com
>>>
--
Jeff Whiting
Qualtrics Senior Software Engineer
jeffw@qualtrics.com
Re: MR hbase export is failing
Posted by Ooh Rong <oo...@gmail.com>.
We had a similar issue on CDH3u4(hbase 0.90) and upgrading to
CDH4(hbase 0.92) solved the problem for us.
If it's an option, I recommend upgrading HBase.
On Wed, Jul 25, 2012 at 4:04 AM, Paul Mackles <pm...@adobe.com> wrote:
> I've seen this when writing when exporting to s3 and assumed it was
> related to write performance. We set hbase.regionserver.lease.period to
> the same values as the task timeout and it helped reduce the # of failures
> though we still get occasional task timeouts. I haven't seen this when
> writing to local HDFS though. Are any of your rows particularly wide? How
> loaded are your data nodes when running the export? Are you running too
> many mappers per node and starving hbase?
>
> On 7/24/12 2:52 PM, "Jeff Whiting" <je...@qualtrics.com> wrote:
>
>>What would cause a scanner timeout exception? Is hdfs too slow? Do I
>>just increase the scanner
>>timeout or is there a better approach.
>>
>>Thanks,
>>~Jeff
>>
>>running: hadoop jar /usr/lib/hbase/hbase-0.90.1-CDH3B4.jar export -D
>>dfs.replication=2 -D
>>mapred.output.compress=true -D
>>mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec
>>-D
>>mapred.output.compression.type=BLOCK response /hbaseBackup/
>>partial/2012-07-20_2012-07-21/response 1 1342742400000 1342828800000
>>org.apache.hadoop.hbase.client.ScannerTimeoutException: 61294ms passed
>>since the last invocation,
>>timeout is currently set to 60000
>> at
>>org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1114)
>> at
>>org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl.nextKeyValue(Table
>>RecordReaderImpl.java:143)
>> at
>>org.apache.hadoop.hbase.mapreduce.TableRecordReader.nextKeyValue(TableReco
>>rdReader.java:142)
>> at
>>org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.nextKeyValue(MapT
>>ask.java:455)
>> at
>>org.apache.hadoop.mapreduce.MapContext.nextKeyValue(MapContext.java:67)
>> at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143)
>> at
>>org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:646)
>> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:322)
>> at org.apache.hadoop.mapred.Child$4.run(Child.java:240)
>> at java.security.AccessController.doPrivileged(Native Method)
>> at javax.security.auth.Subject.doAs(Subject.java:396)
>> at
>>org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.
>>java:1115)
>> at org.apache.hadoop.mapred.Child.main(Child.java:234)
>>Caused by: org.apache.hadoop.hbase.UnknownScannerException:
>>org.apache.hadoop.hbase.UnknownScannerException: Name:
>>-3564751891935236449
>> at
>>org.apache.hadoop.hbase.regionserver.HRegionServer.next(HRegionServer.java
>>:1795)
>> at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
>> at
>>sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorIm
>>pl.java:25)
>> at java.lang.reflect.Method.invoke(Method.java:597)
>> at
>>org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
>> at
>>org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
>>
>> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>Method)
>> at
>>sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAcc
>>essorImpl.java:39)
>> at
>>sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstr
>>uctorAccessorImpl.java:27)
>> at
>>java.lang.reflect.Constructor.newInstance(Constructor.java:513)
>> at
>>org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(Remot
>>eExceptionHandler.java:96)
>> at
>>org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:8
>>3)
>> at
>>org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:3
>>8)
>> at
>>org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementatio
>>n.getRegionServerWithRetries(HConnectionManager.java:1000)
>> at
>>org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1100)
>> ... 12 more
>>
>>--
>>Jeff Whiting
>>Qualtrics Senior Software Engineer
>>jeffw@qualtrics.com
>>
>
Re: MR hbase export is failing
Posted by Paul Mackles <pm...@adobe.com>.
I've seen this when writing when exporting to s3 and assumed it was
related to write performance. We set hbase.regionserver.lease.period to
the same values as the task timeout and it helped reduce the # of failures
though we still get occasional task timeouts. I haven't seen this when
writing to local HDFS though. Are any of your rows particularly wide? How
loaded are your data nodes when running the export? Are you running too
many mappers per node and starving hbase?
On 7/24/12 2:52 PM, "Jeff Whiting" <je...@qualtrics.com> wrote:
>What would cause a scanner timeout exception? Is hdfs too slow? Do I
>just increase the scanner
>timeout or is there a better approach.
>
>Thanks,
>~Jeff
>
>running: hadoop jar /usr/lib/hbase/hbase-0.90.1-CDH3B4.jar export -D
>dfs.replication=2 -D
>mapred.output.compress=true -D
>mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec
>-D
>mapred.output.compression.type=BLOCK response /hbaseBackup/
>partial/2012-07-20_2012-07-21/response 1 1342742400000 1342828800000
>org.apache.hadoop.hbase.client.ScannerTimeoutException: 61294ms passed
>since the last invocation,
>timeout is currently set to 60000
> at
>org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1114)
> at
>org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl.nextKeyValue(Table
>RecordReaderImpl.java:143)
> at
>org.apache.hadoop.hbase.mapreduce.TableRecordReader.nextKeyValue(TableReco
>rdReader.java:142)
> at
>org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.nextKeyValue(MapT
>ask.java:455)
> at
>org.apache.hadoop.mapreduce.MapContext.nextKeyValue(MapContext.java:67)
> at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143)
> at
>org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:646)
> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:322)
> at org.apache.hadoop.mapred.Child$4.run(Child.java:240)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
>org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.
>java:1115)
> at org.apache.hadoop.mapred.Child.main(Child.java:234)
>Caused by: org.apache.hadoop.hbase.UnknownScannerException:
>org.apache.hadoop.hbase.UnknownScannerException: Name:
>-3564751891935236449
> at
>org.apache.hadoop.hbase.regionserver.HRegionServer.next(HRegionServer.java
>:1795)
> at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
> at
>sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorIm
>pl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at
>org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
> at
>org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
>
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>Method)
> at
>sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAcc
>essorImpl.java:39)
> at
>sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstr
>uctorAccessorImpl.java:27)
> at
>java.lang.reflect.Constructor.newInstance(Constructor.java:513)
> at
>org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(Remot
>eExceptionHandler.java:96)
> at
>org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:8
>3)
> at
>org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:3
>8)
> at
>org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementatio
>n.getRegionServerWithRetries(HConnectionManager.java:1000)
> at
>org.apache.hadoop.hbase.client.HTable$ClientScanner.next(HTable.java:1100)
> ... 12 more
>
>--
>Jeff Whiting
>Qualtrics Senior Software Engineer
>jeffw@qualtrics.com
>