You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/04/28 01:12:44 UTC
svn commit: r1097275 [1/8] - in /hbase/trunk: ./ src/docbkx/
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/avro/
src/main/java/org/apache/hadoop/hbase/catalog/
src/main/java/org/apache/hadoop/hbase/client/ src/main/java/o...
Author: stack
Date: Wed Apr 27 23:12:42 2011
New Revision: 1097275
URL: http://svn.apache.org/viewvc?rev=1097275&view=rev
Log:
HBASE-1502 Remove need for heartbeats in HBase
Added:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ServerName.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Addressing.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestHServerInfo.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestServerName.java
Removed:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HMsg.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/BROKE_FIX_TestKillingServersFromMaster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/OOMEHMaster.java
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/docbkx/troubleshooting.xml
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerAddress.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/Server.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/avro/AvroUtil.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HTable.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/rest/StorageClusterStatusResource.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
hbase/trunk/src/main/resources/hbase-webapps/master/master.jsp
hbase/trunk/src/main/resources/hbase-webapps/master/table.jsp
hbase/trunk/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestSerialization.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1097275&r1=1097274&r2=1097275&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Wed Apr 27 23:12:42 2011
@@ -9,6 +9,7 @@ Release 0.91.0 - Unreleased
HBASE-3762 HTableFactory.releaseHTableInterface() should throw IOException
instead of wrapping in RuntimeException (Ted Yu via garyh)
HBASE-3629 Update our thrift to 0.6 (Moaz Reyad)
+ HBASE-1502 Remove need for heartbeats in HBase
BUG FIXES
HBASE-3280 YouAreDeadException being swallowed in HRS getMaster
Modified: hbase/trunk/src/docbkx/troubleshooting.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/docbkx/troubleshooting.xml?rev=1097275&r1=1097274&r2=1097275&view=diff
==============================================================================
--- hbase/trunk/src/docbkx/troubleshooting.xml (original)
+++ hbase/trunk/src/docbkx/troubleshooting.xml Wed Apr 27 23:12:42 2011
@@ -1,419 +1,419 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<chapter version="5.0" xml:id="trouble"
- xmlns="http://docbook.org/ns/docbook"
- xmlns:xlink="http://www.w3.org/1999/xlink"
- xmlns:xi="http://www.w3.org/2001/XInclude"
- xmlns:svg="http://www.w3.org/2000/svg"
- xmlns:m="http://www.w3.org/1998/Math/MathML"
- xmlns:html="http://www.w3.org/1999/xhtml"
- xmlns:db="http://docbook.org/ns/docbook">
- <title>Troubleshooting and Debugging HBase</title>
- <section xml:id="trouble.general">
- <title>General Guidelines</title>
- <para>
- Always start with the master log (TODO: Which lines?).
- Normally itâs just printing the same lines over and over again.
- If not, then thereâs an issue.
- Google or <link xlink:href="http://search-hadoop.com">search-hadoop.com</link>
- should return some hits for those exceptions youâre seeing.
- </para>
- <para>
- An error rarely comes alone in HBase, usually when something gets screwed up what will
- follow may be hundreds of exceptions and stack traces coming from all over the place.
- The best way to approach this type of problem is to walk the log up to where it all
- began, for example one trick with RegionServers is that they will print some
- metrics when aborting so grepping for <emphasis>Dump</emphasis>
- should get you around the start of the problem.
- </para>
- <para>
- RegionServer suicides are ânormalâ, as this is what they do when something goes wrong.
- For example, if ulimit and xcievers (the two most important initial settings, see <xref linkend="ulimit" />)
- arenât changed, it will make it impossible at some point for datanodes to create new threads
- that from the HBase point of view is seen as if HDFS was gone. Think about what would happen if your
- MySQL database was suddenly unable to access files on your local file system, well itâs the same with
- HBase and HDFS. Another very common reason to see RegionServers committing seppuku is when they enter
- prolonged garbage collection pauses that last longer than the default ZooKeeper session timeout.
- For more information on GC pauses, see the
- <link xlink:href="http://www.cloudera.com/blog/2011/02/avoiding-full-gcs-in-hbase-with-memstore-local-allocation-buffers-part-1/">3 part blog post</link> by Todd Lipcon
- and <xref linkend="gcpause" /> above.
- </para>
- </section>
- <section xml:id="trouble.log">
- <title>Logs</title>
- <para>
- The key process logs are as follows... (replace <user> with the user that started the service, and <hostname> for the machine name)
- </para>
- <para>
- NameNode: <filename>$HADOOP_HOME/logs/hadoop-<user>-namenode-<hostname>.log</filename>
- </para>
- <para>
- DataNode: <filename>$HADOOP_HOME/logs/hadoop-<user>-datanode-<hostname>.log</filename>
- </para>
- <para>
- JobTracker: <filename>$HADOOP_HOME/logs/hadoop-<user>-jobtracker-<hostname>.log</filename>
- </para>
- <para>
- TaskTracker: <filename>$HADOOP_HOME/logs/hadoop-<user>-jobtracker-<hostname>.log</filename>
- </para>
- <para>
- HMaster: <filename>$HBASE_HOME/logs/hbase-<user>-master-<hostname>.log</filename>
- </para>
- <para>
- RegionServer: <filename>$HBASE_HOME/logs/hbase-<user>-regionserver-<hostname>.log</filename>
- </para>
- <para>
- ZooKeeper: <filename>TODO</filename>
- </para>
- <section xml:id="trouble.log.locations">
- <title>Log Locations</title>
- <para>For stand-alone deployments the logs are obviously going to be on a single machine, however this is a development configuration only.
- Production deployments need to run on a cluster.</para>
- <section xml:id="trouble.log.locations.namenode">
- <title>NameNode</title>
- <para>The NameNode log is on the NameNode server. The HBase Master is typically run on the NameNode server, and well as ZooKeeper.</para>
- <para>For smaller clusters the JobTracker is typically run on the NameNode server as well.</para>
- </section>
- <section xml:id="trouble.log.locations.datanode">
- <title>DataNode</title>
- <para>Each DataNode server will have a DataNode log for HDFS, as well as a RegionServer log for HBase.</para>
- <para>Additionally, each DataNode server will also have a TaskTracker log for MapReduce task execution.</para>
- </section>
- </section>
-
- </section>
- <section xml:id="trouble.tools">
- <title>Tools</title>
- <section xml:id="trouble.tools.searchhadoop">
- <title>search-hadoop.com</title>
- <para>
- <link xlink:href="http://search-hadoop.com">search-hadoop.com</link> indexes all the mailing lists and <link xlink:href="https://issues.apache.org/jira/browse/HBASE">JIRA</link>, itâs really helpful when looking for Hadoop/HBase-specific issues.
- </para>
- </section>
- <section xml:id="trouble.tools.tail">
- <title>tail</title>
- <para>
- <code>tail</code> is the command line tool that lets you look at the end of a file. Add the â-fâ option and it will refresh when new data is available. Itâs useful when you are wondering whatâs happening, for example, when a cluster is taking a long time to shutdown or startup as you can just fire a new terminal and tail the master log (and maybe a few RegionServers).
- </para>
- </section>
- <section xml:id="trouble.tools.top">
- <title>top</title>
- <para>
- <code>top</code> is probably one of the most important tool when first trying to see whatâs running on a machine and how the resources are consumed. Hereâs an example from production system:
- <programlisting>
-top - 14:46:59 up 39 days, 11:55, 1 user, load average: 3.75, 3.57, 3.84
-Tasks: 309 total, 1 running, 308 sleeping, 0 stopped, 0 zombie
-Cpu(s): 4.5%us, 1.6%sy, 0.0%ni, 91.7%id, 1.4%wa, 0.1%hi, 0.6%si, 0.0%st
-Mem: 24414432k total, 24296956k used, 117476k free, 7196k buffers
-Swap: 16008732k total, 14348k used, 15994384k free, 11106908k cached
-
- PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
-15558 hadoop 18 -2 3292m 2.4g 3556 S 79 10.4 6523:52 java
-13268 hadoop 18 -2 8967m 8.2g 4104 S 21 35.1 5170:30 java
- 8895 hadoop 18 -2 1581m 497m 3420 S 11 2.1 4002:32 java
-â¦
- </programlisting>
- </para>
- <para>
- Here we can see that the system load average during the last five minutes is 3.75, which very roughly means that on average 3.75 threads were waiting for CPU time during these 5 minutes. In general, the âperfectâ utilization equals to the number of cores, under that number the machine is under utilized and over that the machine is over utilized. This is an important concept, see this article to understand it more: <link xlink:href="http://www.linuxjournal.com/article/9001">http://www.linuxjournal.com/article/9001</link>.
- </para>
- <para>
- Apart from load, we can see that the system is using almost all its available RAM but most of it is used for the OS cache (which is good). The swap only has a few KBs in it and this is wanted, high numbers would indicate swapping activity which is the nemesis of performance of Java systems. Another way to detect swapping is when the load average goes through the roof (although this could also be caused by things like a dying disk, among others).
- </para>
- <para>
- The list of processes isnât super useful by default, all we know is that 3 java processes are using about 111% of the CPUs. To know which is which, simply type âcâ and each line will be expanded. Typing â1â will give you the detail of how each CPU is used instead of the average for all of them like shown here.
- </para>
- </section>
- <section xml:id="trouble.tools.jps">
- <title>jps</title>
- <para>
- <code>jps</code> is shipped with every JDK and gives the java process ids for the current user (if root, then it gives the ids for all users). Example:
- <programlisting>
-hadoop@sv4borg12:~$ jps
-1322 TaskTracker
-17789 HRegionServer
-27862 Child
-1158 DataNode
-25115 HQuorumPeer
-2950 Jps
-19750 ThriftServer
-18776 jmx
- </programlisting>
- In order, we see a:
- <itemizedlist>
- <listitem>Hadoop TaskTracker, manages the local Childs</listitem>
- <listitem>HBase RegionServer, serves regions</listitem>
- <listitem>Child, its MapReduce task, cannot tell which type exactly</listitem>
- <listitem>Hadoop TaskTracker, manages the local Childs</listitem>
- <listitem>Hadoop DataNode, serves blocks</listitem>
- <listitem>HQuorumPeer, a zookeeper ensemble member</listitem>
- <listitem>Jps, well⦠itâs the current process</listitem>
- <listitem>ThriftServer, itâs a special one will be running only if thrift was started</listitem>
- <listitem>jmx, this is a local process thatâs part of our monitoring platform ( poorly named maybe). You probably donât have that.</listitem>
- </itemizedlist>
- </para>
- <para>
- You can then do stuff like checking out the full command line that started the process:
- <programlisting>
-hadoop@sv4borg12:~$ ps aux | grep HRegionServer
-hadoop 17789 155 35.2 9067824 8604364 ? S<l Mar04 9855:48 /usr/java/jdk1.6.0_14/bin/java -Xmx8000m -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -XX:+UseConcMarkSweepGC -XX:NewSize=64m -XX:MaxNewSize=64m -XX:CMSInitiatingOccupancyFraction=88 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/export1/hadoop/logs/gc-hbase.log -Dcom.sun.management.jmxremote.port=10102 -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.password.file=/home/hadoop/hbase/conf/jmxremote.password -Dcom.sun.management.jmxremote -Dhbase.log.dir=/export1/hadoop/logs -Dhbase.log.file=hbase-hadoop-regionserver-sv4borg12.log -Dhbase.home.dir=/home/hadoop/hbase -Dhbase.id.str=hadoop -Dhbase.root.logger=INFO,DRFA -Djava.library.path=/home/hadoop/hbase/lib/native/Linux-amd64-64 -classpath /home/hadoop/hbase/bin/../conf:[many jars]:/home/hadoop/hadoop/conf org.apache.hadoop.hbase.regionserver.HRegionServer start
- </programlisting>
- </para>
- </section>
- <section xml:id="trouble.tools.jstack">
- <title>jstack</title>
- <para>
- <code>jstack</code> is one of the most important tools when trying to figure out what a java process is doing apart from looking at the logs. It has to be used in conjunction with jps in order to give it a process id. It shows a list of threads, each one has a name, and they appear in the order that they were created (so the top ones are the most recent threads). Hereâs a few example:
- </para>
- <para>
- The main thread of a RegionServer thatâs waiting for something to do from the master:
- <programlisting>
- "regionserver60020" prio=10 tid=0x0000000040ab4000 nid=0x45cf waiting on condition [0x00007f16b6a96000..0x00007f16b6a96a70]
- java.lang.Thread.State: TIMED_WAITING (parking)
- at sun.misc.Unsafe.park(Native Method)
- - parking to wait for <0x00007f16cd5c2f30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
- at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
- at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
- at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:395)
- at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:647)
- at java.lang.Thread.run(Thread.java:619)
-
- The MemStore flusher thread that is currently flushing to a file:
-"regionserver60020.cacheFlusher" daemon prio=10 tid=0x0000000040f4e000 nid=0x45eb in Object.wait() [0x00007f16b5b86000..0x00007f16b5b87af0]
- java.lang.Thread.State: WAITING (on object monitor)
- at java.lang.Object.wait(Native Method)
- at java.lang.Object.wait(Object.java:485)
- at org.apache.hadoop.ipc.Client.call(Client.java:803)
- - locked <0x00007f16cb14b3a8> (a org.apache.hadoop.ipc.Client$Call)
- at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:221)
- at $Proxy1.complete(Unknown Source)
- at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
- at java.lang.reflect.Method.invoke(Method.java:597)
- at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
- at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
- at $Proxy1.complete(Unknown Source)
- at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.closeInternal(DFSClient.java:3390)
- - locked <0x00007f16cb14b470> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream)
- at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3304)
- at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:61)
- at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:86)
- at org.apache.hadoop.hbase.io.hfile.HFile$Writer.close(HFile.java:650)
- at org.apache.hadoop.hbase.regionserver.StoreFile$Writer.close(StoreFile.java:853)
- at org.apache.hadoop.hbase.regionserver.Store.internalFlushCache(Store.java:467)
- - locked <0x00007f16d00e6f08> (a java.lang.Object)
- at org.apache.hadoop.hbase.regionserver.Store.flushCache(Store.java:427)
- at org.apache.hadoop.hbase.regionserver.Store.access$100(Store.java:80)
- at org.apache.hadoop.hbase.regionserver.Store$StoreFlusherImpl.flushCache(Store.java:1359)
- at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:907)
- at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:834)
- at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:786)
- at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:250)
- at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:224)
- at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:146)
- </programlisting>
- </para>
- <para>
- A handler thread thatâs waiting for stuff to do (like put, delete, scan, etc):
- <programlisting>
-"IPC Server handler 16 on 60020" daemon prio=10 tid=0x00007f16b011d800 nid=0x4a5e waiting on condition [0x00007f16afefd000..0x00007f16afefd9f0]
- java.lang.Thread.State: WAITING (parking)
- at sun.misc.Unsafe.park(Native Method)
- - parking to wait for <0x00007f16cd3f8dd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
- at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
- at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
- at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
- at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
- </programlisting>
- </para>
- <para>
- And one thatâs busy doing an increment of a counter (itâs in the phase where itâs trying to create a scanner in order to read the last value):
- <programlisting>
-"IPC Server handler 66 on 60020" daemon prio=10 tid=0x00007f16b006e800 nid=0x4a90 runnable [0x00007f16acb77000..0x00007f16acb77cf0]
- java.lang.Thread.State: RUNNABLE
- at org.apache.hadoop.hbase.regionserver.KeyValueHeap.<init>(KeyValueHeap.java:56)
- at org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:79)
- at org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:1202)
- at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.<init>(HRegion.java:2209)
- at org.apache.hadoop.hbase.regionserver.HRegion.instantiateInternalScanner(HRegion.java:1063)
- at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1055)
- at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1039)
- at org.apache.hadoop.hbase.regionserver.HRegion.getLastIncrement(HRegion.java:2875)
- at org.apache.hadoop.hbase.regionserver.HRegion.incrementColumnValue(HRegion.java:2978)
- at org.apache.hadoop.hbase.regionserver.HRegionServer.incrementColumnValue(HRegionServer.java:2433)
- at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
- at java.lang.reflect.Method.invoke(Method.java:597)
- at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:560)
- at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1027)
- </programlisting>
- </para>
- <para>
- A thread that receives data from HDFS:
- <programlisting>
-"IPC Client (47) connection to sv4borg9/10.4.24.40:9000 from hadoop" daemon prio=10 tid=0x00007f16a02d0000 nid=0x4fa3 runnable [0x00007f16b517d000..0x00007f16b517dbf0]
- java.lang.Thread.State: RUNNABLE
- at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
- at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215)
- at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
- at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- - locked <0x00007f17d5b68c00> (a sun.nio.ch.Util$1)
- - locked <0x00007f17d5b68be8> (a java.util.Collections$UnmodifiableSet)
- - locked <0x00007f1877959b50> (a sun.nio.ch.EPollSelectorImpl)
- at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
- at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
- at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
- at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
- at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
- at java.io.FilterInputStream.read(FilterInputStream.java:116)
- at org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:304)
- at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
- at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
- - locked <0x00007f1808539178> (a java.io.BufferedInputStream)
- at java.io.DataInputStream.readInt(DataInputStream.java:370)
- at org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:569)
- at org.apache.hadoop.ipc.Client$Connection.run(Client.java:477)
- </programlisting>
- </para>
- <para>
- And here is a master trying to recover a lease after a region server died:
- <programlisting>
-"LeaseChecker" daemon prio=10 tid=0x00000000407ef800 nid=0x76cd waiting on condition [0x00007f6d0eae2000..0x00007f6d0eae2a70]
---
- java.lang.Thread.State: WAITING (on object monitor)
- at java.lang.Object.wait(Native Method)
- at java.lang.Object.wait(Object.java:485)
- at org.apache.hadoop.ipc.Client.call(Client.java:726)
- - locked <0x00007f6d1cd28f80> (a org.apache.hadoop.ipc.Client$Call)
- at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
- at $Proxy1.recoverBlock(Unknown Source)
- at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2636)
- at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.<init>(DFSClient.java:2832)
- at org.apache.hadoop.hdfs.DFSClient.append(DFSClient.java:529)
- at org.apache.hadoop.hdfs.DistributedFileSystem.append(DistributedFileSystem.java:186)
- at org.apache.hadoop.fs.FileSystem.append(FileSystem.java:530)
- at org.apache.hadoop.hbase.util.FSUtils.recoverFileLease(FSUtils.java:619)
- at org.apache.hadoop.hbase.regionserver.wal.HLog.splitLog(HLog.java:1322)
- at org.apache.hadoop.hbase.regionserver.wal.HLog.splitLog(HLog.java:1210)
- at org.apache.hadoop.hbase.master.HMaster.splitLogAfterStartup(HMaster.java:648)
- at org.apache.hadoop.hbase.master.HMaster.joinCluster(HMaster.java:572)
- at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:503)
- </programlisting>
- </para>
- </section>
- <section xml:id="trouble.tools.opentsdb">
- <title>OpenTSDB</title>
- <para>
- <link xlink:href="http://opentsdb.net">OpenTSDB</link> is an excellent alternative to Ganglia as it uses HBase to store all the time series and doesnât have to downsample. Monitoring your own HBase cluster that hosts OpenTSDB is a good exercise.
- </para>
- <para>
- Hereâs an example of a cluster thatâs suffering from hundreds of compactions launched almost all around the same time, which severely affects the IO performance: (TODO: insert graph plotting compactionQueueSize)
- </para>
- <para>
- Itâs a good practice to build dashboards with all the important graphs per machine and per cluster so that debugging issues can be done with a single quick look. For example, at StumbleUpon thereâs one dashboard per cluster with the most important metrics from both the OS and HBase. You can then go down at the machine level and get even more detailed metrics.
- </para>
- </section>
- <section xml:id="trouble.tools.clustersshtop">
- <title>clusterssh+top</title>
- <para>
- clusterssh+top, itâs like a poor manâs monitoring system and it can be quite useful when you have only a few machines as itâs very easy to setup. Starting clusterssh will give you one terminal per machine and another terminal in which whatever you type will be retyped in every window. This means that you can type âtopâ once and it will start it for all of your machines at the same time giving you full view of the current state of your cluster. You can also tail all the logs at the same time, edit files, etc.
- </para>
- </section>
- </section>
-
- <section xml:id="trouble.client">
- <title>Client</title>
- <section xml:id="trouble.client.scantimeout">
- <title>ScannerTimeoutException</title>
- <para>This is thrown if the time between RPC calls from the client to RegionServer exceeds the scan timeout.
- For example, if Scan.setCaching is set to 500, then there will be an RPC call to fetch the next batch of rows every 500 <code>.next()</code> calls on the ResultScanner
- because data is being transferred in blocks of 500 rows to the client. Reducing the setCaching value may be an option, but setting this value too low makes for inefficient
- processing on numbers of rows.
- </para>
- </section>
-
- </section>
- <section xml:id="trouble.rs">
- <title>RegionServer</title>
- <section xml:id="trouble.rs.startup">
- <title>Startup Errors</title>
- <section xml:id="trouble.rs.startup.compression">
- <title>Compression Link Errors</title>
- <para>
- Since compression algorithms such as LZO need to be installed and configured on each cluster this is a frequent source of startup error. If you see messages like this...
- <programlisting>
-11/02/20 01:32:15 ERROR lzo.GPLNativeCodeLoader: Could not load native gpl library
-java.lang.UnsatisfiedLinkError: no gplcompression in java.library.path
- at java.lang.ClassLoader.loadLibrary(ClassLoader.java:1734)
- at java.lang.Runtime.loadLibrary0(Runtime.java:823)
- at java.lang.System.loadLibrary(System.java:1028)
- </programlisting>
- .. then there is a path issue with the compression libraries. See the Configuration section on <link linkend="lzo">LZO compression configuration</link>.
- </para>
- </section>
- </section>
- <section xml:id="trouble.rs.runtime">
- <title>Runtime Errors</title>
-
- <section xml:id="trouble.rs.runtime.filehandles">
- <title>java.io.IOException...(Too many open files)</title>
- <para>
- See the Getting Started section on <link linkend="ulimit">ulimit and nproc configuration</link>.
- </para>
- </section>
- <section xml:id="trouble.rs.runtime.xceivers">
- <title>xceiverCount 258 exceeds the limit of concurrent xcievers 256</title>
- <para>
- This typically shows up in the DataNode logs.
- </para>
- <para>
- See the Getting Started section on <link linkend="dfs.datanode.max.xcievers">xceivers configuration</link>.
- </para>
- </section>
- <section xml:id="trouble.rs.runtime.oom-nt">
- <title>System instability, and the presence of "java.lang.OutOfMemoryError: unable to create new native thread in exceptions" HDFS datanode logs or that of any system daemon</title>
- <para>
- See the Getting Started section on <link linkend="ulimit">ulimit and nproc configuration</link>.
- </para>
- </section>
- <section xml:id="trouble.rs.runtime.gc">
- <title>DFS instability and/or RegionServer lease timeouts</title>
- <para>
- If you see warning messages like this...
- <programlisting>
-2009-02-24 10:01:33,516 WARN org.apache.hadoop.hbase.util.Sleeper: We slept xxx ms, ten times longer than scheduled: 10000
-2009-02-24 10:01:33,516 WARN org.apache.hadoop.hbase.util.Sleeper: We slept xxx ms, ten times longer than scheduled: 15000
-2009-02-24 10:01:36,472 WARN org.apache.hadoop.hbase.regionserver.HRegionServer: unable to report to master for xxx milliseconds - retrying
- </programlisting>
- ... or see full GC compactions then you may be experiencing full GC's.
- </para>
- </section>
- <section xml:id="trouble.rs.runtime.nolivenodes">
- <title>"No live nodes contain current block" and/or YouAreDeadException</title>
- <para>
- These errors can happen either when running out of OS file handles or in periods of severe network problems where the nodes are unreachable.
- </para>
- <para>
- See the Getting Started section on <link linkend="ulimit">ulimit and nproc configuration</link> and check your network.
- </para>
- </section>
-
- </section>
- <section xml:id="trouble.rs.shutdown">
- <title>Shutdown Errors</title>
-
- </section>
-
- </section>
- <section xml:id="trouble.master">
- <title>Master</title>
- <section xml:id="trouble.master.startup">
- <title>Startup Errors</title>
-
- </section>
- <section xml:id="trouble.master.startup">
- <title>Shutdown Errors</title>
-
- </section>
-
- </section>
- </chapter>
+<?xml version="1.0" encoding="UTF-8"?>
+<chapter version="5.0" xml:id="trouble"
+ xmlns="http://docbook.org/ns/docbook"
+ xmlns:xlink="http://www.w3.org/1999/xlink"
+ xmlns:xi="http://www.w3.org/2001/XInclude"
+ xmlns:svg="http://www.w3.org/2000/svg"
+ xmlns:m="http://www.w3.org/1998/Math/MathML"
+ xmlns:html="http://www.w3.org/1999/xhtml"
+ xmlns:db="http://docbook.org/ns/docbook">
+ <title>Troubleshooting and Debugging HBase</title>
+ <section xml:id="trouble.general">
+ <title>General Guidelines</title>
+ <para>
+ Always start with the master log (TODO: Which lines?).
+ Normally itâs just printing the same lines over and over again.
+ If not, then thereâs an issue.
+ Google or <link xlink:href="http://search-hadoop.com">search-hadoop.com</link>
+ should return some hits for those exceptions youâre seeing.
+ </para>
+ <para>
+ An error rarely comes alone in HBase, usually when something gets screwed up what will
+ follow may be hundreds of exceptions and stack traces coming from all over the place.
+ The best way to approach this type of problem is to walk the log up to where it all
+ began, for example one trick with RegionServers is that they will print some
+ metrics when aborting so grepping for <emphasis>Dump</emphasis>
+ should get you around the start of the problem.
+ </para>
+ <para>
+ RegionServer suicides are ânormalâ, as this is what they do when something goes wrong.
+ For example, if ulimit and xcievers (the two most important initial settings, see <xref linkend="ulimit" />)
+ arenât changed, it will make it impossible at some point for datanodes to create new threads
+ that from the HBase point of view is seen as if HDFS was gone. Think about what would happen if your
+ MySQL database was suddenly unable to access files on your local file system, well itâs the same with
+ HBase and HDFS. Another very common reason to see RegionServers committing seppuku is when they enter
+ prolonged garbage collection pauses that last longer than the default ZooKeeper session timeout.
+ For more information on GC pauses, see the
+ <link xlink:href="http://www.cloudera.com/blog/2011/02/avoiding-full-gcs-in-hbase-with-memstore-local-allocation-buffers-part-1/">3 part blog post</link> by Todd Lipcon
+ and <xref linkend="gcpause" /> above.
+ </para>
+ </section>
+ <section xml:id="trouble.log">
+ <title>Logs</title>
+ <para>
+ The key process logs are as follows... (replace <user> with the user that started the service, and <hostname> for the machine name)
+ </para>
+ <para>
+ NameNode: <filename>$HADOOP_HOME/logs/hadoop-<user>-namenode-<hostname>.log</filename>
+ </para>
+ <para>
+ DataNode: <filename>$HADOOP_HOME/logs/hadoop-<user>-datanode-<hostname>.log</filename>
+ </para>
+ <para>
+ JobTracker: <filename>$HADOOP_HOME/logs/hadoop-<user>-jobtracker-<hostname>.log</filename>
+ </para>
+ <para>
+ TaskTracker: <filename>$HADOOP_HOME/logs/hadoop-<user>-jobtracker-<hostname>.log</filename>
+ </para>
+ <para>
+ HMaster: <filename>$HBASE_HOME/logs/hbase-<user>-master-<hostname>.log</filename>
+ </para>
+ <para>
+ RegionServer: <filename>$HBASE_HOME/logs/hbase-<user>-regionserver-<hostname>.log</filename>
+ </para>
+ <para>
+ ZooKeeper: <filename>TODO</filename>
+ </para>
+ <section xml:id="trouble.log.locations">
+ <title>Log Locations</title>
+ <para>For stand-alone deployments the logs are obviously going to be on a single machine, however this is a development configuration only.
+ Production deployments need to run on a cluster.</para>
+ <section xml:id="trouble.log.locations.namenode">
+ <title>NameNode</title>
+ <para>The NameNode log is on the NameNode server. The HBase Master is typically run on the NameNode server, and well as ZooKeeper.</para>
+ <para>For smaller clusters the JobTracker is typically run on the NameNode server as well.</para>
+ </section>
+ <section xml:id="trouble.log.locations.datanode">
+ <title>DataNode</title>
+ <para>Each DataNode server will have a DataNode log for HDFS, as well as a RegionServer log for HBase.</para>
+ <para>Additionally, each DataNode server will also have a TaskTracker log for MapReduce task execution.</para>
+ </section>
+ </section>
+
+ </section>
+ <section xml:id="trouble.tools">
+ <title>Tools</title>
+ <section xml:id="trouble.tools.searchhadoop">
+ <title>search-hadoop.com</title>
+ <para>
+ <link xlink:href="http://search-hadoop.com">search-hadoop.com</link> indexes all the mailing lists and <link xlink:href="https://issues.apache.org/jira/browse/HBASE">JIRA</link>, itâs really helpful when looking for Hadoop/HBase-specific issues.
+ </para>
+ </section>
+ <section xml:id="trouble.tools.tail">
+ <title>tail</title>
+ <para>
+ <code>tail</code> is the command line tool that lets you look at the end of a file. Add the â-fâ option and it will refresh when new data is available. Itâs useful when you are wondering whatâs happening, for example, when a cluster is taking a long time to shutdown or startup as you can just fire a new terminal and tail the master log (and maybe a few RegionServers).
+ </para>
+ </section>
+ <section xml:id="trouble.tools.top">
+ <title>top</title>
+ <para>
+ <code>top</code> is probably one of the most important tool when first trying to see whatâs running on a machine and how the resources are consumed. Hereâs an example from production system:
+ <programlisting>
+top - 14:46:59 up 39 days, 11:55, 1 user, load average: 3.75, 3.57, 3.84
+Tasks: 309 total, 1 running, 308 sleeping, 0 stopped, 0 zombie
+Cpu(s): 4.5%us, 1.6%sy, 0.0%ni, 91.7%id, 1.4%wa, 0.1%hi, 0.6%si, 0.0%st
+Mem: 24414432k total, 24296956k used, 117476k free, 7196k buffers
+Swap: 16008732k total, 14348k used, 15994384k free, 11106908k cached
+
+ PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
+15558 hadoop 18 -2 3292m 2.4g 3556 S 79 10.4 6523:52 java
+13268 hadoop 18 -2 8967m 8.2g 4104 S 21 35.1 5170:30 java
+ 8895 hadoop 18 -2 1581m 497m 3420 S 11 2.1 4002:32 java
+â¦
+ </programlisting>
+ </para>
+ <para>
+ Here we can see that the system load average during the last five minutes is 3.75, which very roughly means that on average 3.75 threads were waiting for CPU time during these 5 minutes. In general, the âperfectâ utilization equals to the number of cores, under that number the machine is under utilized and over that the machine is over utilized. This is an important concept, see this article to understand it more: <link xlink:href="http://www.linuxjournal.com/article/9001">http://www.linuxjournal.com/article/9001</link>.
+ </para>
+ <para>
+ Apart from load, we can see that the system is using almost all its available RAM but most of it is used for the OS cache (which is good). The swap only has a few KBs in it and this is wanted, high numbers would indicate swapping activity which is the nemesis of performance of Java systems. Another way to detect swapping is when the load average goes through the roof (although this could also be caused by things like a dying disk, among others).
+ </para>
+ <para>
+ The list of processes isnât super useful by default, all we know is that 3 java processes are using about 111% of the CPUs. To know which is which, simply type âcâ and each line will be expanded. Typing â1â will give you the detail of how each CPU is used instead of the average for all of them like shown here.
+ </para>
+ </section>
+ <section xml:id="trouble.tools.jps">
+ <title>jps</title>
+ <para>
+ <code>jps</code> is shipped with every JDK and gives the java process ids for the current user (if root, then it gives the ids for all users). Example:
+ <programlisting>
+hadoop@sv4borg12:~$ jps
+1322 TaskTracker
+17789 HRegionServer
+27862 Child
+1158 DataNode
+25115 HQuorumPeer
+2950 Jps
+19750 ThriftServer
+18776 jmx
+ </programlisting>
+ In order, we see a:
+ <itemizedlist>
+ <listitem>Hadoop TaskTracker, manages the local Childs</listitem>
+ <listitem>HBase RegionServer, serves regions</listitem>
+ <listitem>Child, its MapReduce task, cannot tell which type exactly</listitem>
+ <listitem>Hadoop TaskTracker, manages the local Childs</listitem>
+ <listitem>Hadoop DataNode, serves blocks</listitem>
+ <listitem>HQuorumPeer, a zookeeper ensemble member</listitem>
+ <listitem>Jps, well⦠itâs the current process</listitem>
+ <listitem>ThriftServer, itâs a special one will be running only if thrift was started</listitem>
+ <listitem>jmx, this is a local process thatâs part of our monitoring platform ( poorly named maybe). You probably donât have that.</listitem>
+ </itemizedlist>
+ </para>
+ <para>
+ You can then do stuff like checking out the full command line that started the process:
+ <programlisting>
+hadoop@sv4borg12:~$ ps aux | grep HRegionServer
+hadoop 17789 155 35.2 9067824 8604364 ? S<l Mar04 9855:48 /usr/java/jdk1.6.0_14/bin/java -Xmx8000m -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -XX:+UseConcMarkSweepGC -XX:NewSize=64m -XX:MaxNewSize=64m -XX:CMSInitiatingOccupancyFraction=88 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/export1/hadoop/logs/gc-hbase.log -Dcom.sun.management.jmxremote.port=10102 -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.password.file=/home/hadoop/hbase/conf/jmxremote.password -Dcom.sun.management.jmxremote -Dhbase.log.dir=/export1/hadoop/logs -Dhbase.log.file=hbase-hadoop-regionserver-sv4borg12.log -Dhbase.home.dir=/home/hadoop/hbase -Dhbase.id.str=hadoop -Dhbase.root.logger=INFO,DRFA -Djava.library.path=/home/hadoop/hbase/lib/native/Linux-amd64-64 -classpath /home/hadoop/hbase/bin/../conf:[many jars]:/home/hadoop/hadoop/conf org.apache.hadoop.hbase.regionserver.HRegionServer start
+ </programlisting>
+ </para>
+ </section>
+ <section xml:id="trouble.tools.jstack">
+ <title>jstack</title>
+ <para>
+ <code>jstack</code> is one of the most important tools when trying to figure out what a java process is doing apart from looking at the logs. It has to be used in conjunction with jps in order to give it a process id. It shows a list of threads, each one has a name, and they appear in the order that they were created (so the top ones are the most recent threads). Hereâs a few example:
+ </para>
+ <para>
+ The main thread of a RegionServer thatâs waiting for something to do from the master:
+ <programlisting>
+ "regionserver60020" prio=10 tid=0x0000000040ab4000 nid=0x45cf waiting on condition [0x00007f16b6a96000..0x00007f16b6a96a70]
+ java.lang.Thread.State: TIMED_WAITING (parking)
+ at sun.misc.Unsafe.park(Native Method)
+ - parking to wait for <0x00007f16cd5c2f30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
+ at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
+ at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
+ at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:395)
+ at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:647)
+ at java.lang.Thread.run(Thread.java:619)
+
+ The MemStore flusher thread that is currently flushing to a file:
+"regionserver60020.cacheFlusher" daemon prio=10 tid=0x0000000040f4e000 nid=0x45eb in Object.wait() [0x00007f16b5b86000..0x00007f16b5b87af0]
+ java.lang.Thread.State: WAITING (on object monitor)
+ at java.lang.Object.wait(Native Method)
+ at java.lang.Object.wait(Object.java:485)
+ at org.apache.hadoop.ipc.Client.call(Client.java:803)
+ - locked <0x00007f16cb14b3a8> (a org.apache.hadoop.ipc.Client$Call)
+ at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:221)
+ at $Proxy1.complete(Unknown Source)
+ at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
+ at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
+ at java.lang.reflect.Method.invoke(Method.java:597)
+ at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)
+ at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)
+ at $Proxy1.complete(Unknown Source)
+ at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.closeInternal(DFSClient.java:3390)
+ - locked <0x00007f16cb14b470> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream)
+ at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3304)
+ at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:61)
+ at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:86)
+ at org.apache.hadoop.hbase.io.hfile.HFile$Writer.close(HFile.java:650)
+ at org.apache.hadoop.hbase.regionserver.StoreFile$Writer.close(StoreFile.java:853)
+ at org.apache.hadoop.hbase.regionserver.Store.internalFlushCache(Store.java:467)
+ - locked <0x00007f16d00e6f08> (a java.lang.Object)
+ at org.apache.hadoop.hbase.regionserver.Store.flushCache(Store.java:427)
+ at org.apache.hadoop.hbase.regionserver.Store.access$100(Store.java:80)
+ at org.apache.hadoop.hbase.regionserver.Store$StoreFlusherImpl.flushCache(Store.java:1359)
+ at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:907)
+ at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:834)
+ at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:786)
+ at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:250)
+ at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:224)
+ at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:146)
+ </programlisting>
+ </para>
+ <para>
+ A handler thread thatâs waiting for stuff to do (like put, delete, scan, etc):
+ <programlisting>
+"IPC Server handler 16 on 60020" daemon prio=10 tid=0x00007f16b011d800 nid=0x4a5e waiting on condition [0x00007f16afefd000..0x00007f16afefd9f0]
+ java.lang.Thread.State: WAITING (parking)
+ at sun.misc.Unsafe.park(Native Method)
+ - parking to wait for <0x00007f16cd3f8dd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
+ at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
+ at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
+ at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
+ at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
+ </programlisting>
+ </para>
+ <para>
+ And one thatâs busy doing an increment of a counter (itâs in the phase where itâs trying to create a scanner in order to read the last value):
+ <programlisting>
+"IPC Server handler 66 on 60020" daemon prio=10 tid=0x00007f16b006e800 nid=0x4a90 runnable [0x00007f16acb77000..0x00007f16acb77cf0]
+ java.lang.Thread.State: RUNNABLE
+ at org.apache.hadoop.hbase.regionserver.KeyValueHeap.<init>(KeyValueHeap.java:56)
+ at org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:79)
+ at org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:1202)
+ at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.<init>(HRegion.java:2209)
+ at org.apache.hadoop.hbase.regionserver.HRegion.instantiateInternalScanner(HRegion.java:1063)
+ at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1055)
+ at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1039)
+ at org.apache.hadoop.hbase.regionserver.HRegion.getLastIncrement(HRegion.java:2875)
+ at org.apache.hadoop.hbase.regionserver.HRegion.incrementColumnValue(HRegion.java:2978)
+ at org.apache.hadoop.hbase.regionserver.HRegionServer.incrementColumnValue(HRegionServer.java:2433)
+ at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
+ at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
+ at java.lang.reflect.Method.invoke(Method.java:597)
+ at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:560)
+ at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1027)
+ </programlisting>
+ </para>
+ <para>
+ A thread that receives data from HDFS:
+ <programlisting>
+"IPC Client (47) connection to sv4borg9/10.4.24.40:9000 from hadoop" daemon prio=10 tid=0x00007f16a02d0000 nid=0x4fa3 runnable [0x00007f16b517d000..0x00007f16b517dbf0]
+ java.lang.Thread.State: RUNNABLE
+ at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
+ at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215)
+ at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
+ at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
+ - locked <0x00007f17d5b68c00> (a sun.nio.ch.Util$1)
+ - locked <0x00007f17d5b68be8> (a java.util.Collections$UnmodifiableSet)
+ - locked <0x00007f1877959b50> (a sun.nio.ch.EPollSelectorImpl)
+ at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
+ at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
+ at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
+ at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
+ at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
+ at java.io.FilterInputStream.read(FilterInputStream.java:116)
+ at org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:304)
+ at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
+ at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
+ - locked <0x00007f1808539178> (a java.io.BufferedInputStream)
+ at java.io.DataInputStream.readInt(DataInputStream.java:370)
+ at org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:569)
+ at org.apache.hadoop.ipc.Client$Connection.run(Client.java:477)
+ </programlisting>
+ </para>
+ <para>
+ And here is a master trying to recover a lease after a region server died:
+ <programlisting>
+"LeaseChecker" daemon prio=10 tid=0x00000000407ef800 nid=0x76cd waiting on condition [0x00007f6d0eae2000..0x00007f6d0eae2a70]
+--
+ java.lang.Thread.State: WAITING (on object monitor)
+ at java.lang.Object.wait(Native Method)
+ at java.lang.Object.wait(Object.java:485)
+ at org.apache.hadoop.ipc.Client.call(Client.java:726)
+ - locked <0x00007f6d1cd28f80> (a org.apache.hadoop.ipc.Client$Call)
+ at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
+ at $Proxy1.recoverBlock(Unknown Source)
+ at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2636)
+ at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.<init>(DFSClient.java:2832)
+ at org.apache.hadoop.hdfs.DFSClient.append(DFSClient.java:529)
+ at org.apache.hadoop.hdfs.DistributedFileSystem.append(DistributedFileSystem.java:186)
+ at org.apache.hadoop.fs.FileSystem.append(FileSystem.java:530)
+ at org.apache.hadoop.hbase.util.FSUtils.recoverFileLease(FSUtils.java:619)
+ at org.apache.hadoop.hbase.regionserver.wal.HLog.splitLog(HLog.java:1322)
+ at org.apache.hadoop.hbase.regionserver.wal.HLog.splitLog(HLog.java:1210)
+ at org.apache.hadoop.hbase.master.HMaster.splitLogAfterStartup(HMaster.java:648)
+ at org.apache.hadoop.hbase.master.HMaster.joinCluster(HMaster.java:572)
+ at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:503)
+ </programlisting>
+ </para>
+ </section>
+ <section xml:id="trouble.tools.opentsdb">
+ <title>OpenTSDB</title>
+ <para>
+ <link xlink:href="http://opentsdb.net">OpenTSDB</link> is an excellent alternative to Ganglia as it uses HBase to store all the time series and doesnât have to downsample. Monitoring your own HBase cluster that hosts OpenTSDB is a good exercise.
+ </para>
+ <para>
+ Hereâs an example of a cluster thatâs suffering from hundreds of compactions launched almost all around the same time, which severely affects the IO performance: (TODO: insert graph plotting compactionQueueSize)
+ </para>
+ <para>
+ Itâs a good practice to build dashboards with all the important graphs per machine and per cluster so that debugging issues can be done with a single quick look. For example, at StumbleUpon thereâs one dashboard per cluster with the most important metrics from both the OS and HBase. You can then go down at the machine level and get even more detailed metrics.
+ </para>
+ </section>
+ <section xml:id="trouble.tools.clustersshtop">
+ <title>clusterssh+top</title>
+ <para>
+ clusterssh+top, itâs like a poor manâs monitoring system and it can be quite useful when you have only a few machines as itâs very easy to setup. Starting clusterssh will give you one terminal per machine and another terminal in which whatever you type will be retyped in every window. This means that you can type âtopâ once and it will start it for all of your machines at the same time giving you full view of the current state of your cluster. You can also tail all the logs at the same time, edit files, etc.
+ </para>
+ </section>
+ </section>
+
+ <section xml:id="trouble.client">
+ <title>Client</title>
+ <section xml:id="trouble.client.scantimeout">
+ <title>ScannerTimeoutException</title>
+ <para>This is thrown if the time between RPC calls from the client to RegionServer exceeds the scan timeout.
+ For example, if Scan.setCaching is set to 500, then there will be an RPC call to fetch the next batch of rows every 500 <code>.next()</code> calls on the ResultScanner
+ because data is being transferred in blocks of 500 rows to the client. Reducing the setCaching value may be an option, but setting this value too low makes for inefficient
+ processing on numbers of rows.
+ </para>
+ </section>
+
+ </section>
+ <section xml:id="trouble.rs">
+ <title>RegionServer</title>
+ <section xml:id="trouble.rs.startup">
+ <title>Startup Errors</title>
+ <section xml:id="trouble.rs.startup.compression">
+ <title>Compression Link Errors</title>
+ <para>
+ Since compression algorithms such as LZO need to be installed and configured on each cluster this is a frequent source of startup error. If you see messages like this...
+ <programlisting>
+11/02/20 01:32:15 ERROR lzo.GPLNativeCodeLoader: Could not load native gpl library
+java.lang.UnsatisfiedLinkError: no gplcompression in java.library.path
+ at java.lang.ClassLoader.loadLibrary(ClassLoader.java:1734)
+ at java.lang.Runtime.loadLibrary0(Runtime.java:823)
+ at java.lang.System.loadLibrary(System.java:1028)
+ </programlisting>
+ .. then there is a path issue with the compression libraries. See the Configuration section on <link linkend="lzo">LZO compression configuration</link>.
+ </para>
+ </section>
+ </section>
+ <section xml:id="trouble.rs.runtime">
+ <title>Runtime Errors</title>
+
+ <section xml:id="trouble.rs.runtime.filehandles">
+ <title>java.io.IOException...(Too many open files)</title>
+ <para>
+ See the Getting Started section on <link linkend="ulimit">ulimit and nproc configuration</link>.
+ </para>
+ </section>
+ <section xml:id="trouble.rs.runtime.xceivers">
+ <title>xceiverCount 258 exceeds the limit of concurrent xcievers 256</title>
+ <para>
+ This typically shows up in the DataNode logs.
+ </para>
+ <para>
+ See the Getting Started section on <link linkend="dfs.datanode.max.xcievers">xceivers configuration</link>.
+ </para>
+ </section>
+ <section xml:id="trouble.rs.runtime.oom-nt">
+ <title>System instability, and the presence of "java.lang.OutOfMemoryError: unable to create new native thread in exceptions" HDFS datanode logs or that of any system daemon</title>
+ <para>
+ See the Getting Started section on <link linkend="ulimit">ulimit and nproc configuration</link>.
+ </para>
+ </section>
+ <section xml:id="trouble.rs.runtime.gc">
+ <title>DFS instability and/or RegionServer lease timeouts</title>
+ <para>
+ If you see warning messages like this...
+ <programlisting>
+2009-02-24 10:01:33,516 WARN org.apache.hadoop.hbase.util.Sleeper: We slept xxx ms, ten times longer than scheduled: 10000
+2009-02-24 10:01:33,516 WARN org.apache.hadoop.hbase.util.Sleeper: We slept xxx ms, ten times longer than scheduled: 15000
+2009-02-24 10:01:36,472 WARN org.apache.hadoop.hbase.regionserver.HRegionServer: unable to report to master for xxx milliseconds - retrying
+ </programlisting>
+ ... or see full GC compactions then you may be experiencing full GC's.
+ </para>
+ </section>
+ <section xml:id="trouble.rs.runtime.nolivenodes">
+ <title>"No live nodes contain current block" and/or YouAreDeadException</title>
+ <para>
+ These errors can happen either when running out of OS file handles or in periods of severe network problems where the nodes are unreachable.
+ </para>
+ <para>
+ See the Getting Started section on <link linkend="ulimit">ulimit and nproc configuration</link> and check your network.
+ </para>
+ </section>
+
+ </section>
+ <section xml:id="trouble.rs.shutdown">
+ <title>Shutdown Errors</title>
+
+ </section>
+
+ </section>
+ <section xml:id="trouble.master">
+ <title>Master</title>
+ <section xml:id="trouble.master.startup">
+ <title>Startup Errors</title>
+
+ </section>
+ <section xml:id="trouble.master.startup">
+ <title>Shutdown Errors</title>
+
+ </section>
+
+ </section>
+ </chapter>
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java?rev=1097275&r1=1097274&r2=1097275&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java Wed Apr 27 23:12:42 2011
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
@@ -55,13 +56,14 @@ public class ClusterStatus extends Versi
* <dl>
* <dt>0</dt> <dd>initial version</dd>
* <dt>1</dt> <dd>added cluster ID</dd>
+ * <dt>2</dt> <dd>Added Map of ServerName to ServerLoad</dd>
* </dl>
*/
- private static final byte VERSION = 1;
+ private static final byte VERSION = 2;
private String hbaseVersion;
- private Collection<HServerInfo> liveServerInfo;
- private Collection<String> deadServers;
+ private Map<ServerName, HServerLoad> liveServers;
+ private Collection<ServerName> deadServers;
private Map<String, RegionState> intransition;
private String clusterId;
@@ -72,18 +74,28 @@ public class ClusterStatus extends Versi
super();
}
+ public ClusterStatus(final String hbaseVersion, final String clusterid,
+ final Map<ServerName, HServerLoad> servers,
+ final Collection<ServerName> deadServers, final Map<String, RegionState> rit) {
+ this.hbaseVersion = hbaseVersion;
+ this.liveServers = servers;
+ this.deadServers = deadServers;
+ this.intransition = rit;
+ this.clusterId = clusterid;
+ }
+
/**
* @return the names of region servers on the dead list
*/
- public Collection<String> getDeadServerNames() {
+ public Collection<ServerName> getDeadServerNames() {
return Collections.unmodifiableCollection(deadServers);
}
/**
* @return the number of region servers in the cluster
*/
- public int getServers() {
- return liveServerInfo.size();
+ public int getServersSize() {
+ return liveServers.size();
}
/**
@@ -97,11 +109,8 @@ public class ClusterStatus extends Versi
* @return the average cluster load
*/
public double getAverageLoad() {
- int load = 0;
- for (HServerInfo server: liveServerInfo) {
- load += server.getLoad().getLoad();
- }
- return (double)load / (double)liveServerInfo.size();
+ int load = getRegionsCount();
+ return (double)load / (double)getServersSize();
}
/**
@@ -109,8 +118,8 @@ public class ClusterStatus extends Versi
*/
public int getRegionsCount() {
int count = 0;
- for (HServerInfo server: liveServerInfo) {
- count += server.getLoad().getNumberOfRegions();
+ for (Map.Entry<ServerName, HServerLoad> e: this.liveServers.entrySet()) {
+ count += e.getValue().getNumberOfRegions();
}
return count;
}
@@ -120,8 +129,8 @@ public class ClusterStatus extends Versi
*/
public int getRequestsCount() {
int count = 0;
- for (HServerInfo server: liveServerInfo) {
- count += server.getLoad().getNumberOfRequests();
+ for (Map.Entry<ServerName, HServerLoad> e: this.liveServers.entrySet()) {
+ count += e.getValue().getNumberOfRequests();
}
return count;
}
@@ -134,13 +143,6 @@ public class ClusterStatus extends Versi
}
/**
- * @param version the HBase version string
- */
- public void setHBaseVersion(String version) {
- hbaseVersion = version;
- }
-
- /**
* @see java.lang.Object#equals(java.lang.Object)
*/
public boolean equals(Object o) {
@@ -152,7 +154,7 @@ public class ClusterStatus extends Versi
}
return (getVersion() == ((ClusterStatus)o).getVersion()) &&
getHBaseVersion().equals(((ClusterStatus)o).getHBaseVersion()) &&
- liveServerInfo.equals(((ClusterStatus)o).liveServerInfo) &&
+ this.liveServers.equals(((ClusterStatus)o).liveServers) &&
deadServers.equals(((ClusterStatus)o).deadServers);
}
@@ -160,7 +162,7 @@ public class ClusterStatus extends Versi
* @see java.lang.Object#hashCode()
*/
public int hashCode() {
- return VERSION + hbaseVersion.hashCode() + liveServerInfo.hashCode() +
+ return VERSION + hbaseVersion.hashCode() + this.liveServers.hashCode() +
deadServers.hashCode();
}
@@ -175,43 +177,34 @@ public class ClusterStatus extends Versi
/**
* Returns detailed region server information: A list of
- * {@link HServerInfo}, containing server load and resource usage
- * statistics as {@link HServerLoad}, containing per-region
- * statistics as {@link HServerLoad.RegionLoad}.
+ * {@link ServerName}.
* @return region server information
+ * @deprecated Use {@link #getServers()}
*/
- public Collection<HServerInfo> getServerInfo() {
- return Collections.unmodifiableCollection(liveServerInfo);
+ public Collection<ServerName> getServerInfo() {
+ return getServers();
}
- //
- // Setters
- //
-
- public void setServerInfo(Collection<HServerInfo> serverInfo) {
- this.liveServerInfo = serverInfo;
+ public Collection<ServerName> getServers() {
+ return Collections.unmodifiableCollection(this.liveServers.keySet());
}
- public void setDeadServers(Collection<String> deadServers) {
- this.deadServers = deadServers;
+ /**
+ * @param sn
+ * @return Server's load or null if not found.
+ */
+ public HServerLoad getLoad(final ServerName sn) {
+ return this.liveServers.get(sn);
}
public Map<String, RegionState> getRegionsInTransition() {
return this.intransition;
}
- public void setRegionsInTransition(final Map<String, RegionState> m) {
- this.intransition = m;
- }
-
public String getClusterId() {
return clusterId;
}
- public void setClusterId(String id) {
- this.clusterId = id;
- }
-
//
// Writable
//
@@ -219,13 +212,14 @@ public class ClusterStatus extends Versi
public void write(DataOutput out) throws IOException {
super.write(out);
out.writeUTF(hbaseVersion);
- out.writeInt(liveServerInfo.size());
- for (HServerInfo server: liveServerInfo) {
- server.write(out);
+ out.writeInt(getServersSize());
+ for (Map.Entry<ServerName, HServerLoad> e: this.liveServers.entrySet()) {
+ out.writeUTF(e.getKey().toString());
+ e.getValue().write(out);
}
out.writeInt(deadServers.size());
- for (String server: deadServers) {
- out.writeUTF(server);
+ for (ServerName server: deadServers) {
+ out.writeUTF(server.toString());
}
out.writeInt(this.intransition.size());
for (Map.Entry<String, RegionState> e: this.intransition.entrySet()) {
@@ -239,16 +233,17 @@ public class ClusterStatus extends Versi
super.readFields(in);
hbaseVersion = in.readUTF();
int count = in.readInt();
- liveServerInfo = new ArrayList<HServerInfo>(count);
+ this.liveServers = new HashMap<ServerName, HServerLoad>(count);
for (int i = 0; i < count; i++) {
- HServerInfo info = new HServerInfo();
- info.readFields(in);
- liveServerInfo.add(info);
+ String str = in.readUTF();
+ HServerLoad hsl = new HServerLoad();
+ hsl.readFields(in);
+ this.liveServers.put(new ServerName(str), hsl);
}
count = in.readInt();
- deadServers = new ArrayList<String>(count);
+ deadServers = new ArrayList<ServerName>(count);
for (int i = 0; i < count; i++) {
- deadServers.add(in.readUTF());
+ deadServers.add(new ServerName(in.readUTF()));
}
count = in.readInt();
this.intransition = new TreeMap<String, RegionState>();
@@ -260,4 +255,4 @@ public class ClusterStatus extends Versi
}
this.clusterId = in.readUTF();
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1097275&r1=1097274&r2=1097275&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java Wed Apr 27 23:12:42 2011
@@ -373,6 +373,12 @@ public final class HConstants {
/** HBCK special code name used as server name when manipulating ZK nodes */
public static final String HBCK_CODE_NAME = "HBCKServerName";
+ public static final ServerName HBCK_CODE_SERVERNAME =
+ new ServerName(HBCK_CODE_NAME, -1, -1L);
+
+ public static final String KEY_FOR_HOSTNAME_SEEN_BY_MASTER =
+ "hbase.regionserver.hostname.seen.by.master";
+
public static final String HBASE_MASTER_LOGCLEANER_PLUGINS =
"hbase.master.logcleaner.plugins";
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java?rev=1097275&r1=1097274&r2=1097275&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java Wed Apr 27 23:12:42 2011
@@ -19,24 +19,30 @@
*/
package org.apache.hadoop.hbase;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.hbase.util.Addressing;
+
/**
- * Contains the HRegionInfo for the region and the HServerAddress for the
- * HRegionServer serving the region
+ * Data structure to hold HRegionInfo and the address for the hosting
+ * HRegionServer. Immutable.
*/
public class HRegionLocation implements Comparable<HRegionLocation> {
- // TODO: Is this class necessary? Why not just have a Pair?
- private HRegionInfo regionInfo;
- private HServerAddress serverAddress;
+ private final HRegionInfo regionInfo;
+ private final String hostname;
+ private final int port;
/**
* Constructor
- *
* @param regionInfo the HRegionInfo for the region
- * @param serverAddress the HServerAddress for the region server
+ * @param hostname Hostname
+ * @param port port
*/
- public HRegionLocation(HRegionInfo regionInfo, HServerAddress serverAddress) {
+ public HRegionLocation(HRegionInfo regionInfo, final String hostname,
+ final int port) {
this.regionInfo = regionInfo;
- this.serverAddress = serverAddress;
+ this.hostname = hostname;
+ this.port = port;
}
/**
@@ -44,8 +50,8 @@ public class HRegionLocation implements
*/
@Override
public String toString() {
- return "address: " + this.serverAddress.toString() + ", regioninfo: " +
- this.regionInfo.getRegionNameAsString();
+ return "region=" + this.regionInfo.getRegionNameAsString() +
+ ", hostname=" + this.hostname + ", port=" + this.port;
}
/**
@@ -71,7 +77,8 @@ public class HRegionLocation implements
@Override
public int hashCode() {
int result = this.regionInfo.hashCode();
- result ^= this.serverAddress.hashCode();
+ result ^= this.hostname.hashCode();
+ result ^= this.port;
return result;
}
@@ -80,9 +87,30 @@ public class HRegionLocation implements
return regionInfo;
}
- /** @return HServerAddress */
+ /** @return HServerAddress
+ * @deprecated Use {@link #getHostnamePort}
+ */
public HServerAddress getServerAddress(){
- return serverAddress;
+ return new HServerAddress(this.hostname, this.port);
+ }
+
+ public String getHostname() {
+ return this.hostname;
+ }
+
+ public int getPort() {
+ return this.port;
+ }
+
+ /**
+ * @return String made of hostname and port formatted as per {@link Addressing#createHostAndPortStr(String, int)}
+ */
+ public String getHostnamePort() {
+ return Addressing.createHostAndPortStr(this.hostname, this.port);
+ }
+
+ public InetSocketAddress getInetSocketAddress() {
+ return new InetSocketAddress(this.hostname, this.port);
}
//
@@ -91,9 +119,9 @@ public class HRegionLocation implements
public int compareTo(HRegionLocation o) {
int result = this.regionInfo.compareTo(o.regionInfo);
- if(result == 0) {
- result = this.serverAddress.compareTo(o.serverAddress);
- }
- return result;
+ if (result != 0) return result;
+ result = this.hostname.compareTo(o.getHostname());
+ if (result != 0) return result;
+ return this.port - o.getPort();
}
}
\ No newline at end of file
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerAddress.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerAddress.java?rev=1097275&r1=1097274&r2=1097275&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerAddress.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerAddress.java Wed Apr 27 23:12:42 2011
@@ -19,25 +19,38 @@
*/
package org.apache.hadoop.hbase;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.WritableComparable;
-
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.WritableComparable;
/**
- * HServerAddress is a "label" for a HBase server made of host and port number.
+ * HServerAddress hosts a {@link InetSocketAddress} and makes it
+ * {@link WritableComparable}. Resolves on construction AND on
+ * deserialization -- since we're internally creating an InetSocketAddress --
+ * so could end up with different results if the two ends of serialization have
+ * different resolvers. Be careful where you use it. Should only be used when
+ * you need to pass an InetSocketAddress across an RPC. Even then its a bad
+ * idea because of the above resolve issue.
+ * @deprecated Use {@link InetSocketAddress} or {@link ServerName} or
+ * a hostname String and port.
*/
public class HServerAddress implements WritableComparable<HServerAddress> {
- private InetSocketAddress address;
- String stringValue;
+ // Hard to deprecate this class. Its in the API as internal class,
+ // in particular as an inner class of HRegionLocation. Besides, sometimes
+ // we do want to serialize a InetSocketAddress; this class can be used then.
+ private InetSocketAddress address = null;
+ private String cachedToString = "";
+ /**
+ * Constructor for deserialization use only.
+ */
public HServerAddress() {
- this.address = null;
- this.stringValue = null;
+ super();
}
/**
@@ -46,34 +59,20 @@ public class HServerAddress implements W
*/
public HServerAddress(InetSocketAddress address) {
this.address = address;
- this.stringValue = address.getAddress().getHostName() + ":" +
- address.getPort();
checkBindAddressCanBeResolved();
+ this.cachedToString = createCachedToString();
}
- /**
- * @param hostAndPort Hostname and port formatted as <code><hostname> ':' <port></code>
- */
- public HServerAddress(String hostAndPort) {
- int colonIndex = hostAndPort.lastIndexOf(':');
- if (colonIndex < 0) {
- throw new IllegalArgumentException("Not a host:port pair: " + hostAndPort);
- }
- String host = hostAndPort.substring(0, colonIndex);
- int port = Integer.parseInt(hostAndPort.substring(colonIndex + 1));
- this.address = new InetSocketAddress(host, port);
- this.stringValue = address.getHostName() + ":" + port;
- checkBindAddressCanBeResolved();
+ private String createCachedToString() {
+ return this.address.toString();
}
/**
- * @param bindAddress Hostname
+ * @param hostname Hostname
* @param port Port number
*/
- public HServerAddress(String bindAddress, int port) {
- this.address = new InetSocketAddress(bindAddress, port);
- this.stringValue = address.getHostName() + ":" + port;
- checkBindAddressCanBeResolved();
+ public HServerAddress(final String hostname, final int port) {
+ this(new InetSocketAddress(hostname, port));
}
/**
@@ -81,45 +80,48 @@ public class HServerAddress implements W
* @param other HServerAddress to copy from
*/
public HServerAddress(HServerAddress other) {
- String bindAddress = other.getBindAddress();
- int port = other.getPort();
- this.address = new InetSocketAddress(bindAddress, port);
- stringValue = other.stringValue;
- checkBindAddressCanBeResolved();
+ this(new InetSocketAddress(other.getHostname(), other.getPort()));
}
- /** @return Bind address */
+ /** @return Bind address -- the raw IP, the result of a call to
+ * {@link InetSocketAddress#getAddress()#getHostAddress()} --
+ * or null if cannot resolve */
public String getBindAddress() {
- final InetAddress addr = address.getAddress();
- if (addr != null) {
- return addr.getHostAddress();
- } else {
- LogFactory.getLog(HServerAddress.class).error("Could not resolve the"
- + " DNS name of " + stringValue);
- return null;
- }
+ // This returns null if the address is not resolved.
+ final InetAddress addr = this.address.getAddress();
+ if (addr != null) return addr.getHostAddress();
+ LogFactory.getLog(HServerAddress.class).error("Could not resolve the"
+ + " DNS name of " + this.address.toString());
+ return null;
}
private void checkBindAddressCanBeResolved() {
if (getBindAddress() == null) {
throw new IllegalArgumentException("Could not resolve the"
- + " DNS name of " + stringValue);
+ + " DNS name of " + this.address.toString());
}
}
/** @return Port number */
public int getPort() {
- return address.getPort();
+ return this.address.getPort();
}
/** @return Hostname */
public String getHostname() {
- return address.getHostName();
+ return this.address.getHostName();
+ }
+
+ /**
+ * @return Returns <hostname> ':' <port>
+ */
+ public String getHostnameAndPort() {
+ return getHostname() + ":" + getPort();
}
/** @return The InetSocketAddress */
public InetSocketAddress getInetSocketAddress() {
- return address;
+ return this.address;
}
/**
@@ -127,21 +129,15 @@ public class HServerAddress implements W
*/
@Override
public String toString() {
- return stringValue == null ? "" : stringValue;
+ return this.cachedToString;
}
@Override
public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null) {
- return false;
- }
- if (getClass() != o.getClass()) {
- return false;
- }
- return compareTo((HServerAddress) o) == 0;
+ if (this == o) return true;
+ if (o == null) return false;
+ if (getClass() != o.getClass()) return false;
+ return compareTo((HServerAddress)o) == 0;
}
@Override
@@ -158,24 +154,20 @@ public class HServerAddress implements W
public void readFields(DataInput in) throws IOException {
String hostname = in.readUTF();
int port = in.readInt();
-
- if (hostname == null || hostname.length() == 0) {
- address = null;
- stringValue = null;
- } else {
- address = new InetSocketAddress(hostname, port);
- stringValue = hostname + ":" + port;
+ if (hostname != null && hostname.length() > 0) {
+ this.address = new InetSocketAddress(hostname, port);
checkBindAddressCanBeResolved();
+ createCachedToString();
}
}
public void write(DataOutput out) throws IOException {
- if (address == null) {
+ if (this.address == null) {
out.writeUTF("");
out.writeInt(0);
} else {
- out.writeUTF(address.getAddress().getHostName());
- out.writeInt(address.getPort());
+ out.writeUTF(this.address.getAddress().getHostName());
+ out.writeInt(this.address.getPort());
}
}
@@ -187,7 +179,7 @@ public class HServerAddress implements W
// Addresses as Strings may not compare though address is for the one
// server with only difference being that one address has hostname
// resolved whereas other only has IP.
- if (address.equals(o.address)) return 0;
+ if (this.address.equals(o.address)) return 0;
return toString().compareTo(o.toString());
}
-}
+}
\ No newline at end of file