You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hbase.apache.org by "Elliott Clark (JIRA)" <ji...@apache.org> on 2016/02/05 22:30:39 UTC
[jira] [Commented] (HBASE-15222) Use HDR histograms rather than
hadoop or yammer's
[ https://issues.apache.org/jira/browse/HBASE-15222?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15135040#comment-15135040 ]
Elliott Clark commented on HBASE-15222:
---------------------------------------
We recently saw a lot of contention on a loaded server. That contention was almost completely in the metrics subsystem.
{code}
Thread 80 (B.defaultRpcServer.handler=37,queue=11,port=16020):
State: RUNNABLE
Blocked count: 664893
Waited count: 10166003
Stack:
java.lang.ThreadLocal$ThreadLocalMap.expungeStaleEntry(ThreadLocal.java:617)
java.lang.ThreadLocal$ThreadLocalMap.remove(ThreadLocal.java:499)
java.lang.ThreadLocal$ThreadLocalMap.access$200(ThreadLocal.java:298)
java.lang.ThreadLocal.remove(ThreadLocal.java:222)
java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:426)
java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341)
java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881)
com.yammer.metrics.stats.ExponentiallyDecayingSample.unlockForRegularUsage(ExponentiallyDecayingSample.java:196)
com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:113)
com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:81)
org.apache.hadoop.metrics2.lib.MutableHistogram.add(MutableHistogram.java:69)
org.apache.hadoop.metrics2.lib.MutableRangeHistogram.add(MutableRangeHistogram.java:58)
org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceImpl.updateGet(MetricsRegionServerSourceImpl.java:108)
org.apache.hadoop.hbase.regionserver.MetricsRegionServer.updateGet(MetricsRegionServer.java:79)
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:628)
org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2138)
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33656)
org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
Thread 79 (B.defaultRpcServer.handler=36,queue=10,port=16020):
State: RUNNABLE
Blocked count: 546694
Waited count: 8942562
Stack:
java.lang.ThreadLocal$ThreadLocalMap.expungeStaleEntry(ThreadLocal.java:617)
java.lang.ThreadLocal$ThreadLocalMap.remove(ThreadLocal.java:499)
java.lang.ThreadLocal$ThreadLocalMap.access$200(ThreadLocal.java:298)
java.lang.ThreadLocal.remove(ThreadLocal.java:222)
java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:426)
java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341)
java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881)
com.yammer.metrics.stats.ExponentiallyDecayingSample.unlockForRegularUsage(ExponentiallyDecayingSample.java:196)
com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:113)
com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:81)
org.apache.hadoop.metrics2.lib.MutableHistogram.add(MutableHistogram.java:69)
org.apache.hadoop.metrics2.lib.MutableRangeHistogram.add(MutableRangeHistogram.java:58)
org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceImpl.updateGet(MetricsRegionServerSourceImpl.java:108)
org.apache.hadoop.hbase.regionserver.MetricsRegionServer.updateGet(MetricsRegionServer.java:79)
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:628)
org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2138)
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33656)
org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
Thread 78 (B.defaultRpcServer.handler=35,queue=9,port=16020):
State: RUNNABLE
Blocked count: 546871
Waited count: 8886334
Stack:
java.lang.ThreadLocal$ThreadLocalMap.expungeStaleEntry(ThreadLocal.java:617)
java.lang.ThreadLocal$ThreadLocalMap.remove(ThreadLocal.java:499)
java.lang.ThreadLocal$ThreadLocalMap.access$200(ThreadLocal.java:298)
java.lang.ThreadLocal.remove(ThreadLocal.java:222)
java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:426)
java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341)
java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881)
com.yammer.metrics.stats.ExponentiallyDecayingSample.unlockForRegularUsage(ExponentiallyDecayingSample.java:196)
com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:113)
com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:81)
org.apache.hadoop.metrics2.lib.MutableHistogram.add(MutableHistogram.java:69)
org.apache.hadoop.metrics2.lib.MutableRangeHistogram.add(MutableRangeHistogram.java:58)
org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceImpl.updateGet(MetricsRegionServerSourceImpl.java:108)
org.apache.hadoop.hbase.regionserver.MetricsRegionServer.updateGet(MetricsRegionServer.java:79)
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:628)
org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2138)
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33656)
org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
Thread 77 (B.defaultRpcServer.handler=34,queue=8,port=16020):
State: RUNNABLE
Blocked count: 544453
Waited count: 9000337
Stack:
java.lang.ThreadLocal$ThreadLocalMap.expungeStaleEntry(ThreadLocal.java:617)
java.lang.ThreadLocal$ThreadLocalMap.remove(ThreadLocal.java:499)
java.lang.ThreadLocal$ThreadLocalMap.access$200(ThreadLocal.java:298)
java.lang.ThreadLocal.remove(ThreadLocal.java:222)
java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:426)
java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341)
java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881)
com.yammer.metrics.stats.ExponentiallyDecayingSample.unlockForRegularUsage(ExponentiallyDecayingSample.java:196)
com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:113)
com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:81)
org.apache.hadoop.metrics2.lib.MutableHistogram.add(MutableHistogram.java:69)
org.apache.hadoop.metrics2.lib.MutableRangeHistogram.add(MutableRangeHistogram.java:58)
org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceImpl.updateGet(MetricsRegionServerSourceImpl.java:108)
org.apache.hadoop.hbase.regionserver.MetricsRegionServer.updateGet(MetricsRegionServer.java:79)
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:628)
org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2138)
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33656)
org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
Thread 76 (B.defaultRpcServer.handler=33,queue=7,port=16020):
{code}
> Use HDR histograms rather than hadoop or yammer's
> -------------------------------------------------
>
> Key: HBASE-15222
> URL: https://issues.apache.org/jira/browse/HBASE-15222
> Project: HBase
> Issue Type: Bug
> Components: metrics
> Affects Versions: 2.0.0, 1.2.0, 1.3.0
> Reporter: Elliott Clark
> Assignee: Elliott Clark
> Fix For: 2.0.0, 1.3.0
>
>
> Running the benchmarks now, but it looks like the results are pretty extreme. The locking in our histograms is pretty extreme.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)