You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by 陈加俊 <cj...@gmail.com> on 2011/02/28 09:06:43 UTC
java.lang.OutOfMemoryError: Java heap space
My cluster hava 12regionserver and HBase version is 0.20.6 .
AverageLoad: 856.3333333333334
Dead: 0
Live Servers: 12
1-> 192.168.5.152:60020 [requests=86, regions=856, usedHeap=2763, maxHeap=2991]
2-> 192.168.5.146:60020 [requests=48, regions=855, usedHeap=2898, maxHeap=2995]
3-> 192.168.5.147:60020 [requests=1362, regions=856, usedHeap=2748,
maxHeap=2995]
4-> 192.168.5.149:60020 [requests=156, regions=855, usedHeap=2519, maxHeap=2995]
5-> 192.168.5.154:60020 [requests=114, regions=855, usedHeap=2924, maxHeap=2991]
6-> 192.168.5.150:60020 [requests=50, regions=859, usedHeap=2522, maxHeap=2995]
7-> 192.168.5.153:60020 [requests=219, regions=858, usedHeap=2687, maxHeap=2991]
8-> 192.168.5.145:60020 [requests=200, regions=856, usedHeap=2791, maxHeap=2995]
9-> 192.168.5.155:60020 [requests=196, regions=857, usedHeap=2909, maxHeap=2991]
10-> 192.168.5.148:60020 [requests=107, regions=855, usedHeap=2935,
maxHeap=2995]
11-> 192.168.5.157:60020 [requests=74, regions=855, usedHeap=2849, maxHeap=2991]
12-> 192.168.5.156:60020 [requests=53, regions=859, usedHeap=2224, maxHeap=2991]
regions: 10276
Requests Count: 2665
details of regionserver's log when RS shutdown:
2011-02-28 00:24:57,172 WARN org.apache.zookeeper.ClientCnxn: Exception
closing session 0x9b2d886685b60082 to sun.nio.ch.SelectionKeyImpl@60451
3be
java.io.IOException: TIMED OUT
at
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:906)
2011-02-28 00:24:59,674 ERROR
org.apache.hadoop.hbase.regionserver.HRegionServer: Error opening
cjjIndexHtml,http://www.jfdaily.com/c/1_1_y.htm
EC032FC7C4FB2BC0E332082636B3B707,1298250567983
java.lang.OutOfMemoryError: Java heap space
at java.io.BufferedInputStream.<init>(BufferedInputStream.java:178)
at
org.apache.hadoop.hdfs.DFSClient$BlockReader.newBlockReader(DFSClient.java:1380)
at
org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1648)
at
org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:1767)
at java.io.DataInputStream.readFully(DataInputStream.java:178)
at java.io.DataInputStream.readFully(DataInputStream.java:152)
at
org.apache.hadoop.hbase.io.hfile.HFile$FixedFileTrailer.deserialize(HFile.java:1368)
at
org.apache.hadoop.hbase.io.hfile.HFile$Reader.readTrailer(HFile.java:848)
at
org.apache.hadoop.hbase.io.hfile.HFile$Reader.loadFileInfo(HFile.java:793)
at
org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:273)
at
org.apache.hadoop.hbase.regionserver.StoreFile.<init>(StoreFile.java:129)
at
org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:410)
at org.apache.hadoop.hbase.regionserver.Store.<init>(Store.java:221)
at
org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:1636)
at
org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:321)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateRegion(HRegionServer.java:1566)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:1533)
at
org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1453)
at java.lang.Thread.run(Thread.java:619)
2011-02-28 00:24:57,172 WARN org.apache.zookeeper.ClientCnxn: Exception
closing session 0x9a2dd5d1b61e0019 to sun.nio.ch.SelectionKeyImpl@5e9f4
203
java.io.IOException: TIMED OUT
at
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:906)
2011-02-28 00:24:59,696 FATAL
org.apache.hadoop.hbase.regionserver.HRegionServer: OutOfMemoryError,
aborting.
java.lang.OutOfMemoryError: Java heap space
at java.io.BufferedInputStream.<init>(BufferedInputStream.java:178)
at
org.apache.hadoop.hdfs.DFSClient$BlockReader.newBlockReader(DFSClient.java:1380)
at
org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1648)
at
org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:1767)
at java.io.DataInputStream.readFully(DataInputStream.java:178)
at java.io.DataInputStream.readFully(DataInputStream.java:152)
at
org.apache.hadoop.hbase.io.hfile.HFile$FixedFileTrailer.deserialize(HFile.java:1368)
at
org.apache.hadoop.hbase.io.hfile.HFile$Reader.readTrailer(HFile.java:848)
at
org.apache.hadoop.hbase.io.hfile.HFile$Reader.loadFileInfo(HFile.java:793)
at
org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:273)
at
org.apache.hadoop.hbase.regionserver.StoreFile.<init>(StoreFile.java:129)
at
org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:410)
at org.apache.hadoop.hbase.regionserver.Store.<init>(Store.java:221)
at
org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:1636)
at
org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:321)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateRegion(HRegionServer.java:1566)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:1533)
at
org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1453)
at java.lang.Thread.run(Thread.java:619)
2011-02-28 00:24:59,700 INFO org.apache.hadoop.ipc.HBaseServer: IPC Server
handler 8 on 60020, call put([B@69f9aa66,
[Lorg.apache.hadoop.hbase.client.Put;@62244e5) from 192.168.5.179:56884:
error: java.io.IOException: Server not running, aborting
java.io.IOException: Server not running, aborting
at
org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.put(HRegionServer.java:1773)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
2011-02-28 00:24:59,701 INFO org.apache.hadoop.ipc.HBaseServer: IPC Server
handler 18 on 60020, call delete([B@721b4a4f,
row=2011022889990015lmth.12611/0220_1102/3_swen/moc.kjj5.swen//:ptth,
ts=9223372036854775807, families={}) from 192.168.5.161:41682: error:
java.io.IOException: Server not running, aborting
java.io.IOException: Server not running, aborting
at
org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1975)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
2011-02-28 00:24:59,702 INFO org.apache.hadoop.ipc.HBaseServer: IPC Server
handler 2 on 60020, call put([B@5dabc30f,
[Lorg.apache.hadoop.hbase.client.Put;@37933dd4) from 192.168.5.161:41682:
error: java.io.IOException: Server not running, aborting
java.io.IOException: Server not running, aborting
at
org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.put(HRegionServer.java:1773)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
at
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
--
Thanks & Best regards
jiajun
Re: java.lang.OutOfMemoryError: Java heap space
Posted by Stack <st...@duboce.net>.
Thats a lot of regions per regionserver. You should up the size of
your regions if they are default sized. But look in your regionserver
UI. That'll tell you a better story regards how much of your heap is
given over to keeping storefile indices in memory. Can you give hbase
more than this 4G?
St.Ack
On Mon, Feb 28, 2011 at 7:07 PM, 陈加俊 <cj...@gmail.com> wrote:
> I reassigned 4G heap to each RS and add one more RS to the cluster. Now the
> cluster is as follows.I noticed the this line :requests=7, regions=813,
> usedHeap=3823, maxHeap=3991,Is it dangerous signs?
> -------------------------------2011-03-01 11:01:33
> Hbase Version: 0.20.6
> runing : yes
> AverageLoad: 797.0
>
> Dead: 0
>
>
> Live Servers: 13
>
> 1-> 192.168.5.157:60020 [requests=44, regions=812, usedHeap=3082,
> maxHeap=3991]
> 2-> 192.168.5.153:60020 [requests=7, regions=813, usedHeap=3823,
> maxHeap=3991]
> 3-> 192.168.5.148:60020 [requests=63, regions=814, usedHeap=3670,
> maxHeap=3995]
> 4-> 192.168.5.150:60020 [requests=157, regions=816, usedHeap=3185,
> maxHeap=3995]
> 5-> 192.168.5.145:60020 [requests=103, regions=582, usedHeap=2683,
> maxHeap=3995]
> 6-> 192.168.5.156:60020 [requests=36, regions=819, usedHeap=2987,
> maxHeap=3991]
> 7-> 192.168.5.155:60020 [requests=14, regions=812, usedHeap=3458,
> maxHeap=3991]
> 8-> 192.168.5.146:60020 [requests=35, regions=822, usedHeap=2483,
> maxHeap=3995]
> 9-> 192.168.5.152:60020 [requests=10, regions=813, usedHeap=3897,
> maxHeap=3991]
> 10-> 192.168.5.149:60020 [requests=59, regions=821, usedHeap=3377,
> maxHeap=3995]
> 11-> 192.168.5.144:60020 [requests=1155, regions=812, usedHeap=3152,
> maxHeap=3995]
> 12-> 192.168.5.147:60020 [requests=942, regions=813, usedHeap=3411,
> maxHeap=3995]
> 13-> 192.168.5.154:60020 [requests=80, regions=812, usedHeap=2639,
> maxHeap=3991]
>
> regions: 10361
> Requests Count: 2705
> Regions In Transition:
>
> On Tue, Mar 1, 2011 at 1:44 AM, Stack <st...@duboce.net> wrote:
>>
>> Please update to 0.90.1.
>>
>> How much heap have you assigned each server? It looks like its 3G?
>> You have close to 1k regions on each of the nodes. If you look at
>> regionserver UI, at the top, you can see some detail on store files
>> and how much of the heap their indices occupy. How much is it? Is it
>> a significant portion of your heap (See HBASE-3551 for more on what
>> I'm talking about)?
>>
>> Can you give hbase more heap?
>>
>> St.Ack
>>
>>
>> On Mon, Feb 28, 2011 at 12:06 AM, 陈加俊 <cj...@gmail.com> wrote:
>> > My cluster hava 12regionserver and HBase version is 0.20.6 .
>> >
>> > AverageLoad: 856.3333333333334
>> >
>> > Dead: 0
>> >
>> >
>> > Live Servers: 12
>> >
>> > 1-> 192.168.5.152:60020 [requests=86, regions=856, usedHeap=2763,
>> > maxHeap=2991]
>> > 2-> 192.168.5.146:60020 [requests=48, regions=855, usedHeap=2898,
>> > maxHeap=2995]
>> > 3-> 192.168.5.147:60020 [requests=1362, regions=856, usedHeap=2748,
>> > maxHeap=2995]
>> > 4-> 192.168.5.149:60020 [requests=156, regions=855, usedHeap=2519,
>> > maxHeap=2995]
>> > 5-> 192.168.5.154:60020 [requests=114, regions=855, usedHeap=2924,
>> > maxHeap=2991]
>> > 6-> 192.168.5.150:60020 [requests=50, regions=859, usedHeap=2522,
>> > maxHeap=2995]
>> > 7-> 192.168.5.153:60020 [requests=219, regions=858, usedHeap=2687,
>> > maxHeap=2991]
>> > 8-> 192.168.5.145:60020 [requests=200, regions=856, usedHeap=2791,
>> > maxHeap=2995]
>> > 9-> 192.168.5.155:60020 [requests=196, regions=857, usedHeap=2909,
>> > maxHeap=2991]
>> > 10-> 192.168.5.148:60020 [requests=107, regions=855, usedHeap=2935,
>> > maxHeap=2995]
>> > 11-> 192.168.5.157:60020 [requests=74, regions=855, usedHeap=2849,
>> > maxHeap=2991]
>> > 12-> 192.168.5.156:60020 [requests=53, regions=859, usedHeap=2224,
>> > maxHeap=2991]
>> >
>> > regions: 10276
>> > Requests Count: 2665
>> >
>> >
>> > details of regionserver's log when RS shutdown:
>> >
>> >
>> > 2011-02-28 00:24:57,172 WARN org.apache.zookeeper.ClientCnxn: Exception
>> > closing session 0x9b2d886685b60082 to sun.nio.ch.SelectionKeyImpl@60451
>> > 3be
>> > java.io.IOException: TIMED OUT
>> > at
>> > org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:906)
>> > 2011-02-28 00:24:59,674 ERROR
>> > org.apache.hadoop.hbase.regionserver.HRegionServer: Error opening
>> > cjjIndexHtml,http://www.jfdaily.com/c/1_1_y.htm
>> > EC032FC7C4FB2BC0E332082636B3B707,1298250567983
>> > java.lang.OutOfMemoryError: Java heap space
>> > at
>> > java.io.BufferedInputStream.<init>(BufferedInputStream.java:178)
>> > at
>> >
>> > org.apache.hadoop.hdfs.DFSClient$BlockReader.newBlockReader(DFSClient.java:1380)
>> > at
>> >
>> > org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1648)
>> > at
>> >
>> > org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:1767)
>> > at java.io.DataInputStream.readFully(DataInputStream.java:178)
>> > at java.io.DataInputStream.readFully(DataInputStream.java:152)
>> > at
>> >
>> > org.apache.hadoop.hbase.io.hfile.HFile$FixedFileTrailer.deserialize(HFile.java:1368)
>> > at
>> >
>> > org.apache.hadoop.hbase.io.hfile.HFile$Reader.readTrailer(HFile.java:848)
>> > at
>> >
>> > org.apache.hadoop.hbase.io.hfile.HFile$Reader.loadFileInfo(HFile.java:793)
>> > at
>> > org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:273)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.StoreFile.<init>(StoreFile.java:129)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:410)
>> > at
>> > org.apache.hadoop.hbase.regionserver.Store.<init>(Store.java:221)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:1636)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:321)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateRegion(HRegionServer.java:1566)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:1533)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1453)
>> > at java.lang.Thread.run(Thread.java:619)
>> > 2011-02-28 00:24:57,172 WARN org.apache.zookeeper.ClientCnxn: Exception
>> > closing session 0x9a2dd5d1b61e0019 to sun.nio.ch.SelectionKeyImpl@5e9f4
>> > 203
>> > java.io.IOException: TIMED OUT
>> > at
>> > org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:906)
>> > 2011-02-28 00:24:59,696 FATAL
>> > org.apache.hadoop.hbase.regionserver.HRegionServer: OutOfMemoryError,
>> > aborting.
>> > java.lang.OutOfMemoryError: Java heap space
>> > at
>> > java.io.BufferedInputStream.<init>(BufferedInputStream.java:178)
>> > at
>> >
>> > org.apache.hadoop.hdfs.DFSClient$BlockReader.newBlockReader(DFSClient.java:1380)
>> > at
>> >
>> > org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1648)
>> > at
>> >
>> > org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:1767)
>> > at java.io.DataInputStream.readFully(DataInputStream.java:178)
>> > at java.io.DataInputStream.readFully(DataInputStream.java:152)
>> > at
>> >
>> > org.apache.hadoop.hbase.io.hfile.HFile$FixedFileTrailer.deserialize(HFile.java:1368)
>> > at
>> >
>> > org.apache.hadoop.hbase.io.hfile.HFile$Reader.readTrailer(HFile.java:848)
>> > at
>> >
>> > org.apache.hadoop.hbase.io.hfile.HFile$Reader.loadFileInfo(HFile.java:793)
>> > at
>> > org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:273)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.StoreFile.<init>(StoreFile.java:129)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:410)
>> > at
>> > org.apache.hadoop.hbase.regionserver.Store.<init>(Store.java:221)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:1636)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:321)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateRegion(HRegionServer.java:1566)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:1533)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1453)
>> > at java.lang.Thread.run(Thread.java:619)
>> > 2011-02-28 00:24:59,700 INFO org.apache.hadoop.ipc.HBaseServer: IPC
>> > Server
>> > handler 8 on 60020, call put([B@69f9aa66,
>> > [Lorg.apache.hadoop.hbase.client.Put;@62244e5) from 192.168.5.179:56884:
>> > error: java.io.IOException: Server not running, aborting
>> > java.io.IOException: Server not running, aborting
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.put(HRegionServer.java:1773)
>> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> > at
>> >
>> > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>> > at
>> >
>> > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> > at java.lang.reflect.Method.invoke(Method.java:597)
>> > at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
>> > at
>> >
>> > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
>> > 2011-02-28 00:24:59,701 INFO org.apache.hadoop.ipc.HBaseServer: IPC
>> > Server
>> > handler 18 on 60020, call delete([B@721b4a4f,
>> > row=2011022889990015lmth.12611/0220_1102/3_swen/moc.kjj5.swen//:ptth,
>> > ts=9223372036854775807, families={}) from 192.168.5.161:41682: error:
>> > java.io.IOException: Server not running, aborting
>> > java.io.IOException: Server not running, aborting
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1975)
>> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> > at
>> >
>> > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>> > at
>> >
>> > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> > at java.lang.reflect.Method.invoke(Method.java:597)
>> > at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
>> > at
>> >
>> > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
>> > 2011-02-28 00:24:59,702 INFO org.apache.hadoop.ipc.HBaseServer: IPC
>> > Server
>> > handler 2 on 60020, call put([B@5dabc30f,
>> > [Lorg.apache.hadoop.hbase.client.Put;@37933dd4) from
>> > 192.168.5.161:41682:
>> > error: java.io.IOException: Server not running, aborting
>> > java.io.IOException: Server not running, aborting
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
>> > at
>> >
>> > org.apache.hadoop.hbase.regionserver.HRegionServer.put(HRegionServer.java:1773)
>> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> > at
>> >
>> > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>> > at
>> >
>> > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> > at java.lang.reflect.Method.invoke(Method.java:597)
>> > at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
>> > at
>> >
>> > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
>> >
>> > --
>> > Thanks & Best regards
>> > jiajun
>> >
>
>
>
> --
> Thanks & Best regards
> jiajun
>
>
Re: java.lang.OutOfMemoryError: Java heap space
Posted by 陈加俊 <cj...@gmail.com>.
I reassigned 4G heap to each RS and add one more RS to the cluster. Now the
cluster is as follows.I noticed the this line :requests=7, regions=813,
usedHeap=3823, maxHeap=3991,Is it dangerous signs?
-------------------------------2011-03-01 11:01:33
Hbase Version: 0.20.6
runing : yes
AverageLoad: 797.0
Dead: 0
Live Servers: 13
1-> 192.168.5.157:60020 [requests=44, regions=812, usedHeap=3082,
maxHeap=3991]
2-> 192.168.5.153:60020 [requests=7, regions=813, usedHeap=3823,
maxHeap=3991]
3-> 192.168.5.148:60020 [requests=63, regions=814, usedHeap=3670,
maxHeap=3995]
4-> 192.168.5.150:60020 [requests=157, regions=816, usedHeap=3185,
maxHeap=3995]
5-> 192.168.5.145:60020 [requests=103, regions=582, usedHeap=2683,
maxHeap=3995]
6-> 192.168.5.156:60020 [requests=36, regions=819, usedHeap=2987,
maxHeap=3991]
7-> 192.168.5.155:60020 [requests=14, regions=812, usedHeap=3458,
maxHeap=3991]
8-> 192.168.5.146:60020 [requests=35, regions=822, usedHeap=2483,
maxHeap=3995]
9-> 192.168.5.152:60020 [requests=10, regions=813, usedHeap=3897,
maxHeap=3991]
10-> 192.168.5.149:60020 [requests=59, regions=821, usedHeap=3377,
maxHeap=3995]
11-> 192.168.5.144:60020 [requests=1155, regions=812, usedHeap=3152,
maxHeap=3995]
12-> 192.168.5.147:60020 [requests=942, regions=813, usedHeap=3411,
maxHeap=3995]
13-> 192.168.5.154:60020 [requests=80, regions=812, usedHeap=2639,
maxHeap=3991]
regions: 10361
Requests Count: 2705
Regions In Transition:
On Tue, Mar 1, 2011 at 1:44 AM, Stack <st...@duboce.net> wrote:
> Please update to 0.90.1.
>
> How much heap have you assigned each server? It looks like its 3G?
> You have close to 1k regions on each of the nodes. If you look at
> regionserver UI, at the top, you can see some detail on store files
> and how much of the heap their indices occupy. How much is it? Is it
> a significant portion of your heap (See HBASE-3551 for more on what
> I'm talking about)?
>
> Can you give hbase more heap?
>
> St.Ack
>
>
> On Mon, Feb 28, 2011 at 12:06 AM, 陈加俊 <cj...@gmail.com> wrote:
> > My cluster hava 12regionserver and HBase version is 0.20.6 .
> >
> > AverageLoad: 856.3333333333334
> >
> > Dead: 0
> >
> >
> > Live Servers: 12
> >
> > 1-> 192.168.5.152:60020 [requests=86, regions=856, usedHeap=2763,
> maxHeap=2991]
> > 2-> 192.168.5.146:60020 [requests=48, regions=855, usedHeap=2898,
> maxHeap=2995]
> > 3-> 192.168.5.147:60020 [requests=1362, regions=856, usedHeap=2748,
> > maxHeap=2995]
> > 4-> 192.168.5.149:60020 [requests=156, regions=855, usedHeap=2519,
> maxHeap=2995]
> > 5-> 192.168.5.154:60020 [requests=114, regions=855, usedHeap=2924,
> maxHeap=2991]
> > 6-> 192.168.5.150:60020 [requests=50, regions=859, usedHeap=2522,
> maxHeap=2995]
> > 7-> 192.168.5.153:60020 [requests=219, regions=858, usedHeap=2687,
> maxHeap=2991]
> > 8-> 192.168.5.145:60020 [requests=200, regions=856, usedHeap=2791,
> maxHeap=2995]
> > 9-> 192.168.5.155:60020 [requests=196, regions=857, usedHeap=2909,
> maxHeap=2991]
> > 10-> 192.168.5.148:60020 [requests=107, regions=855, usedHeap=2935,
> > maxHeap=2995]
> > 11-> 192.168.5.157:60020 [requests=74, regions=855, usedHeap=2849,
> maxHeap=2991]
> > 12-> 192.168.5.156:60020 [requests=53, regions=859, usedHeap=2224,
> maxHeap=2991]
> >
> > regions: 10276
> > Requests Count: 2665
> >
> >
> > details of regionserver's log when RS shutdown:
> >
> >
> > 2011-02-28 00:24:57,172 WARN org.apache.zookeeper.ClientCnxn: Exception
> > closing session 0x9b2d886685b60082 to sun.nio.ch.SelectionKeyImpl@60451
> > 3be
> > java.io.IOException: TIMED OUT
> > at
> > org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:906)
> > 2011-02-28 00:24:59,674 ERROR
> > org.apache.hadoop.hbase.regionserver.HRegionServer: Error opening
> > cjjIndexHtml,http://www.jfdaily.com/c/1_1_y.htm
> > EC032FC7C4FB2BC0E332082636B3B707,1298250567983
> > java.lang.OutOfMemoryError: Java heap space
> > at
> java.io.BufferedInputStream.<init>(BufferedInputStream.java:178)
> > at
> >
> org.apache.hadoop.hdfs.DFSClient$BlockReader.newBlockReader(DFSClient.java:1380)
> > at
> >
> org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1648)
> > at
> > org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:1767)
> > at java.io.DataInputStream.readFully(DataInputStream.java:178)
> > at java.io.DataInputStream.readFully(DataInputStream.java:152)
> > at
> >
> org.apache.hadoop.hbase.io.hfile.HFile$FixedFileTrailer.deserialize(HFile.java:1368)
> > at
> > org.apache.hadoop.hbase.io.hfile.HFile$Reader.readTrailer(HFile.java:848)
> > at
> >
> org.apache.hadoop.hbase.io.hfile.HFile$Reader.loadFileInfo(HFile.java:793)
> > at
> > org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:273)
> > at
> > org.apache.hadoop.hbase.regionserver.StoreFile.<init>(StoreFile.java:129)
> > at
> > org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:410)
> > at
> org.apache.hadoop.hbase.regionserver.Store.<init>(Store.java:221)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:1636)
> > at
> > org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:321)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateRegion(HRegionServer.java:1566)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:1533)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1453)
> > at java.lang.Thread.run(Thread.java:619)
> > 2011-02-28 00:24:57,172 WARN org.apache.zookeeper.ClientCnxn: Exception
> > closing session 0x9a2dd5d1b61e0019 to sun.nio.ch.SelectionKeyImpl@5e9f4
> > 203
> > java.io.IOException: TIMED OUT
> > at
> > org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:906)
> > 2011-02-28 00:24:59,696 FATAL
> > org.apache.hadoop.hbase.regionserver.HRegionServer: OutOfMemoryError,
> > aborting.
> > java.lang.OutOfMemoryError: Java heap space
> > at
> java.io.BufferedInputStream.<init>(BufferedInputStream.java:178)
> > at
> >
> org.apache.hadoop.hdfs.DFSClient$BlockReader.newBlockReader(DFSClient.java:1380)
> > at
> >
> org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1648)
> > at
> > org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:1767)
> > at java.io.DataInputStream.readFully(DataInputStream.java:178)
> > at java.io.DataInputStream.readFully(DataInputStream.java:152)
> > at
> >
> org.apache.hadoop.hbase.io.hfile.HFile$FixedFileTrailer.deserialize(HFile.java:1368)
> > at
> > org.apache.hadoop.hbase.io.hfile.HFile$Reader.readTrailer(HFile.java:848)
> > at
> >
> org.apache.hadoop.hbase.io.hfile.HFile$Reader.loadFileInfo(HFile.java:793)
> > at
> > org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:273)
> > at
> > org.apache.hadoop.hbase.regionserver.StoreFile.<init>(StoreFile.java:129)
> > at
> > org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:410)
> > at
> org.apache.hadoop.hbase.regionserver.Store.<init>(Store.java:221)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:1636)
> > at
> > org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:321)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateRegion(HRegionServer.java:1566)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:1533)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1453)
> > at java.lang.Thread.run(Thread.java:619)
> > 2011-02-28 00:24:59,700 INFO org.apache.hadoop.ipc.HBaseServer: IPC
> Server
> > handler 8 on 60020, call put([B@69f9aa66,
> > [Lorg.apache.hadoop.hbase.client.Put;@62244e5) from 192.168.5.179:56884:
> > error: java.io.IOException: Server not running, aborting
> > java.io.IOException: Server not running, aborting
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.put(HRegionServer.java:1773)
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> > at java.lang.reflect.Method.invoke(Method.java:597)
> > at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
> > at
> > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
> > 2011-02-28 00:24:59,701 INFO org.apache.hadoop.ipc.HBaseServer: IPC
> Server
> > handler 18 on 60020, call delete([B@721b4a4f,
> > row=2011022889990015lmth.12611/0220_1102/3_swen/moc.kjj5.swen//:ptth,
> > ts=9223372036854775807, families={}) from 192.168.5.161:41682: error:
> > java.io.IOException: Server not running, aborting
> > java.io.IOException: Server not running, aborting
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1975)
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> > at java.lang.reflect.Method.invoke(Method.java:597)
> > at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
> > at
> > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
> > 2011-02-28 00:24:59,702 INFO org.apache.hadoop.ipc.HBaseServer: IPC
> Server
> > handler 2 on 60020, call put([B@5dabc30f,
> > [Lorg.apache.hadoop.hbase.client.Put;@37933dd4) from 192.168.5.161:41682
> :
> > error: java.io.IOException: Server not running, aborting
> > java.io.IOException: Server not running, aborting
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
> > at
> >
> org.apache.hadoop.hbase.regionserver.HRegionServer.put(HRegionServer.java:1773)
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> > at java.lang.reflect.Method.invoke(Method.java:597)
> > at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
> > at
> > org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
> >
> > --
> > Thanks & Best regards
> > jiajun
> >
>
--
Thanks & Best regards
jiajun
Re: java.lang.OutOfMemoryError: Java heap space
Posted by Stack <st...@duboce.net>.
Please update to 0.90.1.
How much heap have you assigned each server? It looks like its 3G?
You have close to 1k regions on each of the nodes. If you look at
regionserver UI, at the top, you can see some detail on store files
and how much of the heap their indices occupy. How much is it? Is it
a significant portion of your heap (See HBASE-3551 for more on what
I'm talking about)?
Can you give hbase more heap?
St.Ack
On Mon, Feb 28, 2011 at 12:06 AM, 陈加俊 <cj...@gmail.com> wrote:
> My cluster hava 12regionserver and HBase version is 0.20.6 .
>
> AverageLoad: 856.3333333333334
>
> Dead: 0
>
>
> Live Servers: 12
>
> 1-> 192.168.5.152:60020 [requests=86, regions=856, usedHeap=2763, maxHeap=2991]
> 2-> 192.168.5.146:60020 [requests=48, regions=855, usedHeap=2898, maxHeap=2995]
> 3-> 192.168.5.147:60020 [requests=1362, regions=856, usedHeap=2748,
> maxHeap=2995]
> 4-> 192.168.5.149:60020 [requests=156, regions=855, usedHeap=2519, maxHeap=2995]
> 5-> 192.168.5.154:60020 [requests=114, regions=855, usedHeap=2924, maxHeap=2991]
> 6-> 192.168.5.150:60020 [requests=50, regions=859, usedHeap=2522, maxHeap=2995]
> 7-> 192.168.5.153:60020 [requests=219, regions=858, usedHeap=2687, maxHeap=2991]
> 8-> 192.168.5.145:60020 [requests=200, regions=856, usedHeap=2791, maxHeap=2995]
> 9-> 192.168.5.155:60020 [requests=196, regions=857, usedHeap=2909, maxHeap=2991]
> 10-> 192.168.5.148:60020 [requests=107, regions=855, usedHeap=2935,
> maxHeap=2995]
> 11-> 192.168.5.157:60020 [requests=74, regions=855, usedHeap=2849, maxHeap=2991]
> 12-> 192.168.5.156:60020 [requests=53, regions=859, usedHeap=2224, maxHeap=2991]
>
> regions: 10276
> Requests Count: 2665
>
>
> details of regionserver's log when RS shutdown:
>
>
> 2011-02-28 00:24:57,172 WARN org.apache.zookeeper.ClientCnxn: Exception
> closing session 0x9b2d886685b60082 to sun.nio.ch.SelectionKeyImpl@60451
> 3be
> java.io.IOException: TIMED OUT
> at
> org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:906)
> 2011-02-28 00:24:59,674 ERROR
> org.apache.hadoop.hbase.regionserver.HRegionServer: Error opening
> cjjIndexHtml,http://www.jfdaily.com/c/1_1_y.htm
> EC032FC7C4FB2BC0E332082636B3B707,1298250567983
> java.lang.OutOfMemoryError: Java heap space
> at java.io.BufferedInputStream.<init>(BufferedInputStream.java:178)
> at
> org.apache.hadoop.hdfs.DFSClient$BlockReader.newBlockReader(DFSClient.java:1380)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1648)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:1767)
> at java.io.DataInputStream.readFully(DataInputStream.java:178)
> at java.io.DataInputStream.readFully(DataInputStream.java:152)
> at
> org.apache.hadoop.hbase.io.hfile.HFile$FixedFileTrailer.deserialize(HFile.java:1368)
> at
> org.apache.hadoop.hbase.io.hfile.HFile$Reader.readTrailer(HFile.java:848)
> at
> org.apache.hadoop.hbase.io.hfile.HFile$Reader.loadFileInfo(HFile.java:793)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:273)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.<init>(StoreFile.java:129)
> at
> org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:410)
> at org.apache.hadoop.hbase.regionserver.Store.<init>(Store.java:221)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:1636)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:321)
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateRegion(HRegionServer.java:1566)
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:1533)
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1453)
> at java.lang.Thread.run(Thread.java:619)
> 2011-02-28 00:24:57,172 WARN org.apache.zookeeper.ClientCnxn: Exception
> closing session 0x9a2dd5d1b61e0019 to sun.nio.ch.SelectionKeyImpl@5e9f4
> 203
> java.io.IOException: TIMED OUT
> at
> org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:906)
> 2011-02-28 00:24:59,696 FATAL
> org.apache.hadoop.hbase.regionserver.HRegionServer: OutOfMemoryError,
> aborting.
> java.lang.OutOfMemoryError: Java heap space
> at java.io.BufferedInputStream.<init>(BufferedInputStream.java:178)
> at
> org.apache.hadoop.hdfs.DFSClient$BlockReader.newBlockReader(DFSClient.java:1380)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1648)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:1767)
> at java.io.DataInputStream.readFully(DataInputStream.java:178)
> at java.io.DataInputStream.readFully(DataInputStream.java:152)
> at
> org.apache.hadoop.hbase.io.hfile.HFile$FixedFileTrailer.deserialize(HFile.java:1368)
> at
> org.apache.hadoop.hbase.io.hfile.HFile$Reader.readTrailer(HFile.java:848)
> at
> org.apache.hadoop.hbase.io.hfile.HFile$Reader.loadFileInfo(HFile.java:793)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:273)
> at
> org.apache.hadoop.hbase.regionserver.StoreFile.<init>(StoreFile.java:129)
> at
> org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:410)
> at org.apache.hadoop.hbase.regionserver.Store.<init>(Store.java:221)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:1636)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:321)
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateRegion(HRegionServer.java:1566)
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:1533)
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1453)
> at java.lang.Thread.run(Thread.java:619)
> 2011-02-28 00:24:59,700 INFO org.apache.hadoop.ipc.HBaseServer: IPC Server
> handler 8 on 60020, call put([B@69f9aa66,
> [Lorg.apache.hadoop.hbase.client.Put;@62244e5) from 192.168.5.179:56884:
> error: java.io.IOException: Server not running, aborting
> java.io.IOException: Server not running, aborting
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.put(HRegionServer.java:1773)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
> 2011-02-28 00:24:59,701 INFO org.apache.hadoop.ipc.HBaseServer: IPC Server
> handler 18 on 60020, call delete([B@721b4a4f,
> row=2011022889990015lmth.12611/0220_1102/3_swen/moc.kjj5.swen//:ptth,
> ts=9223372036854775807, families={}) from 192.168.5.161:41682: error:
> java.io.IOException: Server not running, aborting
> java.io.IOException: Server not running, aborting
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1975)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
> 2011-02-28 00:24:59,702 INFO org.apache.hadoop.ipc.HBaseServer: IPC Server
> handler 2 on 60020, call put([B@5dabc30f,
> [Lorg.apache.hadoop.hbase.client.Put;@37933dd4) from 192.168.5.161:41682:
> error: java.io.IOException: Server not running, aborting
> java.io.IOException: Server not running, aborting
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:2307)
> at
> org.apache.hadoop.hbase.regionserver.HRegionServer.put(HRegionServer.java:1773)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:657)
> at
> org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915)
>
> --
> Thanks & Best regards
> jiajun
>