You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by Gaojinchao <ga...@huawei.com> on 2011/04/28 12:00:14 UTC
About parameter
In my test cluster. It can't assign Meta table.(one Hmaster and two region server).
I find assigned meta region timed out and reopened.
I think we should set default value (hbase.master.assignment.timeoutmonitor.timeout) bigger.
2011-04-14 11:48:19,240 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Received request to open region: .META.,,1.1028785192
2011-04-14 11:48:19,252 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Processing open of .META.,,1.1028785192
2011-04-14 11:48:19,257 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempting to transition node 1028785192/.META. from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2011-04-14 11:48:19,291 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Successfully transitioned node 1028785192 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2011-04-14 11:48:19,311 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Opening region: REGION => {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192, TABLE => {{NAME => '.META.', IS_META => 'true', FAMILIES => [{NAME => 'info', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '10', COMPRESSION => 'NONE', TTL => '2147483647', BLOCKSIZE => '8192', IN_MEMORY => 'true', BLOCKCACHE => 'true'}]}}
2011-04-14 11:48:19,814 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Instantiated .META.,,1.1028785192
2011-04-14 11:48:21,297 DEBUG org.apache.hadoop.hbase.regionserver.Store: loaded hdfs://10.18.52.108:9000/hbase/.META./1028785192/info/3950786077714265980, isReference=false, isBulkLoadResult=false, seqid=204, majorCompaction=false
2011-04-14 11:48:21,796 INFO org.apache.hadoop.hbase.regionserver.HRegion: Replaying edits from hdfs://10.18.52.108:9000/hbase/.META./1028785192/recovered.edits/0000000000000000207; minSequenceid=204
2011-04-14 11:48:59,243 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Received request to open region: .META.,,1.1028785192
2011-04-14 11:49:22,391 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempting to transition node 1028785192/.META. from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2011-04-14 11:49:22,396 WARN org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempt to transition the unassigned node for 1028785192 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING failed, the node existed but was version 102 not the expected version 101
2011-04-14 11:49:22,396 WARN org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Failed refreshing OPENING; region=1028785192, context=open_region_progress
2011-04-14 11:49:22,396 WARN org.apache.hadoop.hbase.regionserver.HRegion: Progressable reporter failed, stopping replay
2011-04-14 11:49:22,447 ERROR org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Failed open of region=.META.,,1.1028785192
java.io.IOException: Progressable reporter failed, stopping replay
at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:1903)
at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:1828)
at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:353)
at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:2546)
at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:2532)
at org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:262)
at org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:94)
at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:151)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
2011-04-14 11:49:22,452 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Processing open of .META.,,1.1028785192
2011-04-14 11:49:22,452 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempting to transition node 1028785192/.META. from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2011-04-14 11:49:22,523 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Successfully transitioned node 1028785192 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2011-04-14 11:49:22,524 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Opening region: REGION => {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192, TABLE => {{NAME => '.META.', IS_META => 'true', FAMILIES => [{NAME => 'info', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '10', COMPRESSION => 'NONE', TTL => '2147483647', BLOCKSIZE => '8192', IN_MEMORY => 'true', BLOCKCACHE => 'true'}]}}
2011-04-14 11:49:22,830 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Instantiated .META.,,1.1028785192
2011-04-14 11:49:24,194 DEBUG org.apache.hadoop.hbase.regionserver.Store: loaded hdfs://10.18.52.108:9000/hbase/.META./1028785192/info/3950786077714265980, isReference=false, isBulkLoadResult=false, seqid=204, majorCompaction=false
2011-04-14 11:49:24,520 INFO org.apache.hadoop.hbase.regionserver.HRegion: Replaying edits from hdfs://10.18.52.108:9000/hbase/.META./1028785192/recovered.edits/0000000000000000207; minSequenceid=204
2011-04-14 11:49:59,234 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Received request to open region: .META.,,1.1028785192
2011-04-14 11:50:19,620 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempting to transition node 1028785192/.META. from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2011-04-14 11:50:19,624 WARN org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempt to transition the unassigned node for 1028785192 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING failed, the node existed but was version 104 not the expected version 103
2011-04-14 11:50:19,624 WARN org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Failed refreshing OPENING; region=1028785192, context=open_region_progress
2011-04-14 11:50:19,624 WARN org.apache.hadoop.hbase.regionserver.HRegion: Progressable reporter failed, stopping replay
2011-04-14 11:50:19,662 ERROR org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Failed open of region=.META.,,1.1028785192
java.io.IOException: Progressable reporter failed, stopping replay
at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:1903)
at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:1828)
at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:353)
at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:2546)
at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:2532)
at org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:262)
at org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:94)
at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:151)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
2011-04-14 11:50:19,663 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Processing open of .META.,,1.1028785192
Re: About parameter
Posted by Jean-Daniel Cryans <jd...@apache.org>.
The issue you saw could have been mitigated by:
https://issues.apache.org/jira/browse/HBASE-3741
Take also into account this bug:
https://issues.apache.org/jira/browse/HBASE-3669
J-D
On Thu, Apr 28, 2011 at 3:00 AM, Gaojinchao <ga...@huawei.com> wrote:
> In my test cluster. It can't assign Meta table.(one Hmaster and two region server).
> I find assigned meta region timed out and reopened.
>
> I think we should set default value (hbase.master.assignment.timeoutmonitor.timeout) bigger.
>
> 2011-04-14 11:48:19,240 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Received request to open region: .META.,,1.1028785192
> 2011-04-14 11:48:19,252 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Processing open of .META.,,1.1028785192
> 2011-04-14 11:48:19,257 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempting to transition node 1028785192/.META. from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
> 2011-04-14 11:48:19,291 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Successfully transitioned node 1028785192 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
> 2011-04-14 11:48:19,311 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Opening region: REGION => {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192, TABLE => {{NAME => '.META.', IS_META => 'true', FAMILIES => [{NAME => 'info', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '10', COMPRESSION => 'NONE', TTL => '2147483647', BLOCKSIZE => '8192', IN_MEMORY => 'true', BLOCKCACHE => 'true'}]}}
> 2011-04-14 11:48:19,814 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Instantiated .META.,,1.1028785192
> 2011-04-14 11:48:21,297 DEBUG org.apache.hadoop.hbase.regionserver.Store: loaded hdfs://10.18.52.108:9000/hbase/.META./1028785192/info/3950786077714265980, isReference=false, isBulkLoadResult=false, seqid=204, majorCompaction=false
> 2011-04-14 11:48:21,796 INFO org.apache.hadoop.hbase.regionserver.HRegion: Replaying edits from hdfs://10.18.52.108:9000/hbase/.META./1028785192/recovered.edits/0000000000000000207; minSequenceid=204
> 2011-04-14 11:48:59,243 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Received request to open region: .META.,,1.1028785192
> 2011-04-14 11:49:22,391 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempting to transition node 1028785192/.META. from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
> 2011-04-14 11:49:22,396 WARN org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempt to transition the unassigned node for 1028785192 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING failed, the node existed but was version 102 not the expected version 101
> 2011-04-14 11:49:22,396 WARN org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Failed refreshing OPENING; region=1028785192, context=open_region_progress
> 2011-04-14 11:49:22,396 WARN org.apache.hadoop.hbase.regionserver.HRegion: Progressable reporter failed, stopping replay
> 2011-04-14 11:49:22,447 ERROR org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Failed open of region=.META.,,1.1028785192
> java.io.IOException: Progressable reporter failed, stopping replay
> at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:1903)
> at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:1828)
> at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:353)
> at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:2546)
> at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:2532)
> at org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:262)
> at org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:94)
> at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:151)
> at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:619)
> 2011-04-14 11:49:22,452 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Processing open of .META.,,1.1028785192
> 2011-04-14 11:49:22,452 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempting to transition node 1028785192/.META. from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
> 2011-04-14 11:49:22,523 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Successfully transitioned node 1028785192 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
> 2011-04-14 11:49:22,524 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Opening region: REGION => {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192, TABLE => {{NAME => '.META.', IS_META => 'true', FAMILIES => [{NAME => 'info', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '10', COMPRESSION => 'NONE', TTL => '2147483647', BLOCKSIZE => '8192', IN_MEMORY => 'true', BLOCKCACHE => 'true'}]}}
> 2011-04-14 11:49:22,830 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Instantiated .META.,,1.1028785192
> 2011-04-14 11:49:24,194 DEBUG org.apache.hadoop.hbase.regionserver.Store: loaded hdfs://10.18.52.108:9000/hbase/.META./1028785192/info/3950786077714265980, isReference=false, isBulkLoadResult=false, seqid=204, majorCompaction=false
> 2011-04-14 11:49:24,520 INFO org.apache.hadoop.hbase.regionserver.HRegion: Replaying edits from hdfs://10.18.52.108:9000/hbase/.META./1028785192/recovered.edits/0000000000000000207; minSequenceid=204
> 2011-04-14 11:49:59,234 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Received request to open region: .META.,,1.1028785192
> 2011-04-14 11:50:19,620 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempting to transition node 1028785192/.META. from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
> 2011-04-14 11:50:19,624 WARN org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:60020-0x12f52766dcd0014 Attempt to transition the unassigned node for 1028785192 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING failed, the node existed but was version 104 not the expected version 103
> 2011-04-14 11:50:19,624 WARN org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Failed refreshing OPENING; region=1028785192, context=open_region_progress
> 2011-04-14 11:50:19,624 WARN org.apache.hadoop.hbase.regionserver.HRegion: Progressable reporter failed, stopping replay
> 2011-04-14 11:50:19,662 ERROR org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Failed open of region=.META.,,1.1028785192
> java.io.IOException: Progressable reporter failed, stopping replay
> at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:1903)
> at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:1828)
> at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:353)
> at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:2546)
> at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:2532)
> at org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:262)
> at org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:94)
> at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:151)
> at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:619)
> 2011-04-14 11:50:19,663 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Processing open of .META.,,1.1028785192
>
>