You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hbase.apache.org by "Enis Soztutar (JIRA)" <ji...@apache.org> on 2016/04/08 01:45:25 UTC
[jira] [Commented] (HBASE-15613) TestNamespaceCommand times out
[ https://issues.apache.org/jira/browse/HBASE-15613?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15231350#comment-15231350 ]
Enis Soztutar commented on HBASE-15613:
---------------------------------------
This is the relevant stack trace.
{code}
"hconnection-0x166b3616-metaLookup-shared--pool118-t1" #647 daemon prio=5 os_prio=31 tid=0x00007fd485469000 nid=0x1e327 in Object.wait() [0x0000000140e1f000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:460)
at io.netty.util.concurrent.DefaultPromise.await0(DefaultPromise.java:355)
- locked <0x0000000774a8cca0> (a org.apache.hadoop.hbase.ipc.AsyncCall)
at io.netty.util.concurrent.DefaultPromise.await(DefaultPromise.java:266)
at io.netty.util.concurrent.AbstractFuture.get(AbstractFuture.java:42)
at org.apache.hadoop.hbase.ipc.AsyncRpcClient.call(AsyncRpcClient.java:248)
at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:35264)
at org.apache.hadoop.hbase.client.ClientSmallScanner$SmallScannerCallable.call(ClientSmallScanner.java:200)
at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:1)
at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:165)
at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:360)
at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:1)
at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:99)
at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:65)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"hconnection-0x79d11914-shared-pool115-t1" #644 daemon prio=5 os_prio=31 tid=0x00007fd4830c8000 nid=0x2261f in Object.wait() [0x000000014066c000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:460)
at io.netty.util.concurrent.DefaultPromise.await0(DefaultPromise.java:355)
- locked <0x0000000774a34908> (a org.apache.hadoop.hbase.ipc.AsyncCall)
at io.netty.util.concurrent.DefaultPromise.await(DefaultPromise.java:266)
at io.netty.util.concurrent.AbstractFuture.get(AbstractFuture.java:42)
at org.apache.hadoop.hbase.ipc.AsyncRpcClient.call(AsyncRpcClient.java:248)
at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.multi(ClientProtos.java:35312)
at org.apache.hadoop.hbase.client.MultiServerCallable.call(MultiServerCallable.java:125)
at org.apache.hadoop.hbase.client.MultiServerCallable.call(MultiServerCallable.java:1)
at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:165)
at org.apache.hadoop.hbase.client.AsyncProcess$AsyncRequestFutureImpl$SingleServerRequestRunnable.run(AsyncProcess.java:746)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"PriorityRpcServer.handler=3,queue=1,port=53673" #161 daemon prio=5 os_prio=31 tid=0x00007fd4868af000 nid=0x10203 in Object.wait() [0x00000001360de000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:460)
at java.util.concurrent.TimeUnit.timedWait(TimeUnit.java:348)
at org.apache.hadoop.hbase.client.ResultBoundedCompletionService.poll(ResultBoundedCompletionService.java:159)
- locked <0x00000007746290b8> (a [Lorg.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture;)
at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:198)
at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:1)
at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:165)
at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.loadCache(ClientSmallReversedScanner.java:212)
at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.next(ClientSmallReversedScanner.java:186)
at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegionInMeta(ConnectionImplementation.java:831)
at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:737)
at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:721)
at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:678)
at org.apache.hadoop.hbase.client.ConnectionImplementation.getRegionLocation(ConnectionImplementation.java:517)
at org.apache.hadoop.hbase.client.HRegionLocator.getRegionLocation(HRegionLocator.java:81)
at org.apache.hadoop.hbase.client.RegionServerCallable.prepare(RegionServerCallable.java:75)
at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:97)
at org.apache.hadoop.hbase.client.HTable.get(HTable.java:442)
at org.apache.hadoop.hbase.client.HTable.get(HTable.java:408)
at org.apache.hadoop.hbase.security.access.AccessControlLists.getPermissions(AccessControlLists.java:465)
at org.apache.hadoop.hbase.security.access.AccessController.updateACL(AccessController.java:278)
at org.apache.hadoop.hbase.security.access.AccessController.postPut(AccessController.java:1663)
at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$30.call(RegionCoprocessorHost.java:930)
at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1645)
at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1721)
at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1677)
at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postPut(RegionCoprocessorHost.java:926)
at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:3252)
at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2940)
at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2882)
at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:867)
at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:829)
at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2305)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:34826)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2250)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:112)
at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:137)
at org.apache.hadoop.hbase.ipc.RpcExecutor$2.run(RpcExecutor.java:112)
at java.lang.Thread.run(Thread.java:745)
"PriorityRpcServer.handler=1,queue=1,port=53673" #159 daemon prio=5 os_prio=31 tid=0x00007fd4858a6000 nid=0xfe03 in Object.wait() [0x0000000135863000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at org.apache.hadoop.hbase.client.AsyncProcess.waitForMaximumCurrentTasks(AsyncProcess.java:1775)
- locked <0x00000007749db900> (a java.util.concurrent.atomic.AtomicLong)
at org.apache.hadoop.hbase.client.AsyncProcess.waitForAllPreviousOpsAndReset(AsyncProcess.java:1804)
at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:241)
at org.apache.hadoop.hbase.client.BufferedMutatorImpl.flush(BufferedMutatorImpl.java:191)
- locked <0x00000007749da838> (a org.apache.hadoop.hbase.client.BufferedMutatorImpl)
at org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:973)
at org.apache.hadoop.hbase.client.HTable.put(HTable.java:563)
at org.apache.hadoop.hbase.security.access.AccessControlLists.addUserPermission(AccessControlLists.java:177)
at org.apache.hadoop.hbase.security.access.AccessController$8.run(AccessController.java:2218)
at org.apache.hadoop.hbase.security.access.AccessController$8.run(AccessController.java:1)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.security.SecurityUtil.doAsUser(SecurityUtil.java:448)
at org.apache.hadoop.security.SecurityUtil.doAsLoginUser(SecurityUtil.java:429)
at sun.reflect.GeneratedMethodAccessor45.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.apache.hadoop.hbase.util.Methods.call(Methods.java:39)
at org.apache.hadoop.hbase.security.User.runAsLoginUser(User.java:183)
at org.apache.hadoop.hbase.security.access.AccessController.grant(AccessController.java:2215)
at org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos$AccessControlService$1.grant(AccessControlProtos.java:9933)
at org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos$AccessControlService.callMethod(AccessControlProtos.java:10097)
at org.apache.hadoop.hbase.regionserver.HRegion.execService(HRegion.java:7503)
at org.apache.hadoop.hbase.regionserver.RSRpcServices.execServiceOnRegion(RSRpcServices.java:2087)
at org.apache.hadoop.hbase.regionserver.RSRpcServices.execService(RSRpcServices.java:2069)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:34822)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2250)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:112)
at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:137)
at org.apache.hadoop.hbase.ipc.RpcExecutor$2.run(RpcExecutor.java:112)
at java.lang.Thread.run(Thread.java:745)
"main" #1 prio=5 os_prio=31 tid=0x00007fd481802000 nid=0x1303 in Object.wait() [0x000000010f267000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:460)
at io.netty.util.concurrent.DefaultPromise.await0(DefaultPromise.java:355)
- locked <0x000000076db8ead8> (a org.apache.hadoop.hbase.ipc.AsyncCall)
at io.netty.util.concurrent.DefaultPromise.await(DefaultPromise.java:266)
at io.netty.util.concurrent.AbstractFuture.get(AbstractFuture.java:42)
at org.apache.hadoop.hbase.ipc.AsyncRpcClient.call(AsyncRpcClient.java:248)
at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.execService(ClientProtos.java:35288)
at org.apache.hadoop.hbase.protobuf.ProtobufUtil.execService(ProtobufUtil.java:1599)
at org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel$1.call(RegionCoprocessorRpcChannel.java:102)
at org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel$1.call(RegionCoprocessorRpcChannel.java:1)
at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:99)
at org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel.callExecService(RegionCoprocessorRpcChannel.java:106)
at org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel.callBlockingMethod(CoprocessorRpcChannel.java:73)
at org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos$AccessControlService$BlockingStub.grant(AccessControlProtos.java:10280)
at org.apache.hadoop.hbase.protobuf.ProtobufUtil.grant(ProtobufUtil.java:2194)
at org.apache.hadoop.hbase.security.access.SecureTestUtil$2.call(SecureTestUtil.java:375)
at org.apache.hadoop.hbase.security.access.SecureTestUtil$2.call(SecureTestUtil.java:1)
at org.apache.hadoop.hbase.security.access.SecureTestUtil.updateACLs(SecureTestUtil.java:332)
at org.apache.hadoop.hbase.security.access.SecureTestUtil.grantGlobal(SecureTestUtil.java:367)
at org.apache.hadoop.hbase.security.access.TestNamespaceCommands.beforeClass(TestNamespaceCommands.java:179)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)
at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:675)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)
{code}
There does not seem to be a deadlock, but the threads just hang around. Will dig more when I have time.
> TestNamespaceCommand times out
> ------------------------------
>
> Key: HBASE-15613
> URL: https://issues.apache.org/jira/browse/HBASE-15613
> Project: HBase
> Issue Type: Bug
> Reporter: Enis Soztutar
> Assignee: Enis Soztutar
>
> We think that the root cause maybe HBASE-15295. Will inspect more.
> See
> https://issues.apache.org/jira/browse/HBASE-15537?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)