You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@kylin.apache.org by 《秦殇》！健 <ye...@foxmail.com> on 2017/03/03 04:47:27 UTC
build kylin cube occur hbase error
Hi, All. I build cube occur a lot of error. my env is kylin1.6+cdh5.7, my cdh hbase is cluster.
kylin 1.6 xx.xx.xx.32cdh1(hbase) xx.xx.xx.32 cdh2(hbase) xx.xx.xx.33cdh3(hbase) xx.xx.xx.34as follows:
2017-03-02 22:41:48,066 ERROR [pool-8-thread-1] dao.ExecutableDao:148 : error get all Jobs:org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=6, exceptions:Thu Mar 02 22:41:48 GMT+08:00 2017, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=114431: row '/execute/' on table 'kylin_metadata' at region=kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309., hostname=hf1-cdh1,60020,1488424273046, seqNum=2
at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:276) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:207) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320) at org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:295) at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:160) at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:155) at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:867) at org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:137) at org.apache.kylin.storage.hbase.HBaseResourceStore.listResourcesImpl(HBaseResourceStore.java:107) at org.apache.kylin.common.persistence.ResourceStore.listResources(ResourceStore.java:121) at org.apache.kylin.job.dao.ExecutableDao.getJobIds(ExecutableDao.java:138) at org.apache.kylin.job.manager.ExecutableManager.getAllJobIds(ExecutableManager.java:207) at org.apache.kylin.job.impl.threadpool.DefaultScheduler$FetcherRunner.run(DefaultScheduler.java:85) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745)Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=114431: row '/execute/' on table 'kylin_metadata' at region=kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309., hostname=hf1-cdh1,60020,1488424273046, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:65) ... 3 moreCaused by: java.net.ConnectException: Connection refused at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:744) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:530) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:416) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:722) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:906) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:873) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1242) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:394) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:203) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:64) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:360) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:334) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126) ... 4 more
Then I found one of my hbase regionserver is shutdown, bad is kylin and hbase regionserver in common machine.
I restart my regionserver. one other error occur:
Caused by: org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: Region kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309. is not online on hf1-cdh1,60020,1488505545946 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2920) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1053) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2385) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33648) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108) at java.lang.Thread.run(Thread.java:745)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:526) at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106) at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:327) at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:402) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:203) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:64) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:360) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:334) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126) ... 4 moreCaused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.NotServingRegionException): org.apache.hadoop.hbase.NotServingRegionException: Region kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309. is not online on hf1-cdh1,60020,1488505545946 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2920) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1053) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2385) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33648) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108) at java.lang.Thread.run(Thread.java:745)
at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1268) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:394) ... 10 more
Could you give me a help, my kylin service can not to use!!!!
回复： build kylin cube occur hbase error
Posted by 《秦殇》！健 <ye...@foxmail.com>.
thx,I will check my log careful。: )
------------------ 原始邮件 ------------------
发件人: "RuiFeng Shan";<sh...@gmail.com>;
发送时间: 2017年3月3日(星期五) 下午5:58
收件人: "dev"<de...@kylin.apache.org>;
主题: Re: build kylin cube occur hbase error
Caused by: org.apache.hadoop.hbase.NotServingRegionException:
org.apache.hadoop.hbase.NotServingRegionException: Region
kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309. is not
online on hf1-cdh1,60020,1488505545946
I think that it is beacuse your hbase regionserver failed to restart.
Firstly, you can observe the hbase web ui to make sure whether all hbase
service(master and regionserver) work well or not.
Secondly, you can find error info from hbase log files.
If hbase service work well, And then, you can restart your kylin service.
2017-03-03 12:47 GMT+08:00 《秦殇》！健 <ye...@foxmail.com>:
> Hi, All. I build cube occur a lot of error. my env is kylin1.6+cdh5.7, my
> cdh hbase is cluster.
> kylin 1.6 xx.xx.xx.32cdh1(hbase) xx.xx.xx.32 cdh2(hbase)
> xx.xx.xx.33cdh3(hbase) xx.xx.xx.34as follows:
>
> 2017-03-02 22:41:48,066 ERROR [pool-8-thread-1] dao.ExecutableDao:148 :
> error get all Jobs:org.apache.hadoop.hbase.client.RetriesExhaustedException:
> Failed after attempts=6, exceptions:Thu Mar 02 22:41:48 GMT+08:00 2017,
> null, java.net.SocketTimeoutException: callTimeout=60000,
> callDuration=114431: row '/execute/' on table 'kylin_metadata' at
> region=kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309.,
> hostname=hf1-cdh1,60020,1488424273046, seqNum=2
> at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadRepli
> cas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:276)
> at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(
> ScannerCallableWithReplicas.java:207) at org.apache.hadoop.hbase.
> client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60)
> at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
> at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320)
> at org.apache.hadoop.hbase.client.ClientScanner.
> nextScanner(ClientScanner.java:295) at org.apache.hadoop.hbase.
> client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:160)
> at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:155)
> at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:867)
> at org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:137)
> at org.apache.kylin.storage.hbase.HBaseResourceStore.listResourcesImpl(HBaseResourceStore.java:107)
> at org.apache.kylin.common.persistence.ResourceStore.
> listResources(ResourceStore.java:121) at org.apache.kylin.job.dao.
> ExecutableDao.getJobIds(ExecutableDao.java:138) at
> org.apache.kylin.job.manager.ExecutableManager.getAllJobIds(ExecutableManager.java:207)
> at org.apache.kylin.job.impl.threadpool.DefaultScheduler$
> FetcherRunner.run(DefaultScheduler.java:85) at java.util.concurrent.
> Executors$RunnableAdapter.call(Executors.java:471) at
> java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) at
> java.util.concurrent.ScheduledThreadPoolExecutor$
> ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
> at java.util.concurrent.ScheduledThreadPoolExecutor$
> ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)Caused by: java.net.SocketTimeoutException:
> callTimeout=60000, callDuration=114431: row '/execute/' on table
> 'kylin_metadata' at region=kylin_metadata,,1488427068259.
> 599efd68884035a133f306657bafb309., hostname=hf1-cdh1,60020,1488424273046,
> seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCaller.
> callWithRetries(RpcRetryingCaller.java:159) at org.apache.hadoop.hbase.
> client.ResultBoundedCompletionService$QueueingFuture.run(
> ResultBoundedCompletionService.java:65) ... 3 moreCaused by:
> java.net.ConnectException: Connection refused at
> sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:744)
> at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:530) at
> org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494) at
> org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.
> setupConnection(RpcClientImpl.java:416) at org.apache.hadoop.hbase.ipc.
> RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:722) at
> org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.
> writeRequest(RpcClientImpl.java:906) at org.apache.hadoop.hbase.ipc.
> RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:873)
> at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1242)
> at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
> at org.apache.hadoop.hbase.ipc.AbstractRpcClient$
> BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
> at org.apache.hadoop.hbase.protobuf.generated.
> ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094)
> at org.apache.hadoop.hbase.client.ScannerCallable.
> openScanner(ScannerCallable.java:394) at org.apache.hadoop.hbase.
> client.ScannerCallable.call(ScannerCallable.java:203) at
> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:64)
> at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
> at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$
> RetryingRPC.call(ScannerCallableWithReplicas.java:360) at
> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$
> RetryingRPC.call(ScannerCallableWithReplicas.java:334) at
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
> ... 4 more
>
> Then I found one of my hbase regionserver is shutdown, bad is kylin and
> hbase regionserver in common machine.
>
>
> I restart my regionserver. one other error occur:
> Caused by: org.apache.hadoop.hbase.NotServingRegionException:
> org.apache.hadoop.hbase.NotServingRegionException: Region
> kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309. is not
> online on hf1-cdh1,60020,1488505545946 at org.apache.hadoop.hbase.
> regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2920)
> at org.apache.hadoop.hbase.regionserver.RSRpcServices.
> getRegion(RSRpcServices.java:1053) at org.apache.hadoop.hbase.
> regionserver.RSRpcServices.scan(RSRpcServices.java:2385) at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.
> callBlockingMethod(ClientProtos.java:33648) at
> org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170) at
> org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109) at
> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
> at java.lang.Thread.run(Thread.java:745)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
> Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(
> NativeConstructorAccessorImpl.java:57) at sun.reflect.
> DelegatingConstructorAccessorImpl.newInstance(
> DelegatingConstructorAccessorImpl.java:45) at
> java.lang.reflect.Constructor.newInstance(Constructor.java:526) at
> org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
> at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
> at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:327)
> at org.apache.hadoop.hbase.client.ScannerCallable.
> openScanner(ScannerCallable.java:402) at org.apache.hadoop.hbase.
> client.ScannerCallable.call(ScannerCallable.java:203) at
> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:64)
> at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
> at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$
> RetryingRPC.call(ScannerCallableWithReplicas.java:360) at
> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$
> RetryingRPC.call(ScannerCallableWithReplicas.java:334) at
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
> ... 4 moreCaused by: org.apache.hadoop.hbase.ipc.
> RemoteWithExtrasException(org.apache.hadoop.hbase.NotServingRegionException):
> org.apache.hadoop.hbase.NotServingRegionException: Region
> kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309. is not
> online on hf1-cdh1,60020,1488505545946 at org.apache.hadoop.hbase.
> regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2920)
> at org.apache.hadoop.hbase.regionserver.RSRpcServices.
> getRegion(RSRpcServices.java:1053) at org.apache.hadoop.hbase.
> regionserver.RSRpcServices.scan(RSRpcServices.java:2385) at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.
> callBlockingMethod(ClientProtos.java:33648) at
> org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170) at
> org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109) at
> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
> at java.lang.Thread.run(Thread.java:745)
> at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1268)
> at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
> at org.apache.hadoop.hbase.ipc.AbstractRpcClient$
> BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
> at org.apache.hadoop.hbase.protobuf.generated.
> ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094)
> at org.apache.hadoop.hbase.client.ScannerCallable.
> openScanner(ScannerCallable.java:394) ... 10 more
> Could you give me a help, my kylin service can not to use!!!!
--
Re: build kylin cube occur hbase error
Posted by RuiFeng Shan <sh...@gmail.com>.
Caused by: org.apache.hadoop.hbase.NotServingRegionException:
org.apache.hadoop.hbase.NotServingRegionException: Region
kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309. is not
online on hf1-cdh1,60020,1488505545946
I think that it is beacuse your hbase regionserver failed to restart.
Firstly, you can observe the hbase web ui to make sure whether all hbase
service(master and regionserver) work well or not.
Secondly, you can find error info from hbase log files.
If hbase service work well, And then, you can restart your kylin service.
2017-03-03 12:47 GMT+08:00 《秦殇》！健 <ye...@foxmail.com>:
> Hi, All. I build cube occur a lot of error. my env is kylin1.6+cdh5.7, my
> cdh hbase is cluster.
> kylin 1.6 xx.xx.xx.32cdh1(hbase) xx.xx.xx.32 cdh2(hbase)
> xx.xx.xx.33cdh3(hbase) xx.xx.xx.34as follows:
>
> 2017-03-02 22:41:48,066 ERROR [pool-8-thread-1] dao.ExecutableDao:148 :
> error get all Jobs:org.apache.hadoop.hbase.client.RetriesExhaustedException:
> Failed after attempts=6, exceptions:Thu Mar 02 22:41:48 GMT+08:00 2017,
> null, java.net.SocketTimeoutException: callTimeout=60000,
> callDuration=114431: row '/execute/' on table 'kylin_metadata' at
> region=kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309.,
> hostname=hf1-cdh1,60020,1488424273046, seqNum=2
> at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadRepli
> cas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:276)
> at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(
> ScannerCallableWithReplicas.java:207) at org.apache.hadoop.hbase.
> client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60)
> at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
> at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320)
> at org.apache.hadoop.hbase.client.ClientScanner.
> nextScanner(ClientScanner.java:295) at org.apache.hadoop.hbase.
> client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:160)
> at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:155)
> at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:867)
> at org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:137)
> at org.apache.kylin.storage.hbase.HBaseResourceStore.listResourcesImpl(HBaseResourceStore.java:107)
> at org.apache.kylin.common.persistence.ResourceStore.
> listResources(ResourceStore.java:121) at org.apache.kylin.job.dao.
> ExecutableDao.getJobIds(ExecutableDao.java:138) at
> org.apache.kylin.job.manager.ExecutableManager.getAllJobIds(ExecutableManager.java:207)
> at org.apache.kylin.job.impl.threadpool.DefaultScheduler$
> FetcherRunner.run(DefaultScheduler.java:85) at java.util.concurrent.
> Executors$RunnableAdapter.call(Executors.java:471) at
> java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) at
> java.util.concurrent.ScheduledThreadPoolExecutor$
> ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
> at java.util.concurrent.ScheduledThreadPoolExecutor$
> ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)Caused by: java.net.SocketTimeoutException:
> callTimeout=60000, callDuration=114431: row '/execute/' on table
> 'kylin_metadata' at region=kylin_metadata,,1488427068259.
> 599efd68884035a133f306657bafb309., hostname=hf1-cdh1,60020,1488424273046,
> seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCaller.
> callWithRetries(RpcRetryingCaller.java:159) at org.apache.hadoop.hbase.
> client.ResultBoundedCompletionService$QueueingFuture.run(
> ResultBoundedCompletionService.java:65) ... 3 moreCaused by:
> java.net.ConnectException: Connection refused at
> sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:744)
> at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:530) at
> org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494) at
> org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.
> setupConnection(RpcClientImpl.java:416) at org.apache.hadoop.hbase.ipc.
> RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:722) at
> org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.
> writeRequest(RpcClientImpl.java:906) at org.apache.hadoop.hbase.ipc.
> RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:873)
> at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1242)
> at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
> at org.apache.hadoop.hbase.ipc.AbstractRpcClient$
> BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
> at org.apache.hadoop.hbase.protobuf.generated.
> ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094)
> at org.apache.hadoop.hbase.client.ScannerCallable.
> openScanner(ScannerCallable.java:394) at org.apache.hadoop.hbase.
> client.ScannerCallable.call(ScannerCallable.java:203) at
> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:64)
> at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
> at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$
> RetryingRPC.call(ScannerCallableWithReplicas.java:360) at
> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$
> RetryingRPC.call(ScannerCallableWithReplicas.java:334) at
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
> ... 4 more
>
> Then I found one of my hbase regionserver is shutdown, bad is kylin and
> hbase regionserver in common machine.
>
>
> I restart my regionserver. one other error occur:
> Caused by: org.apache.hadoop.hbase.NotServingRegionException:
> org.apache.hadoop.hbase.NotServingRegionException: Region
> kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309. is not
> online on hf1-cdh1,60020,1488505545946 at org.apache.hadoop.hbase.
> regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2920)
> at org.apache.hadoop.hbase.regionserver.RSRpcServices.
> getRegion(RSRpcServices.java:1053) at org.apache.hadoop.hbase.
> regionserver.RSRpcServices.scan(RSRpcServices.java:2385) at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.
> callBlockingMethod(ClientProtos.java:33648) at
> org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170) at
> org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109) at
> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
> at java.lang.Thread.run(Thread.java:745)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
> Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(
> NativeConstructorAccessorImpl.java:57) at sun.reflect.
> DelegatingConstructorAccessorImpl.newInstance(
> DelegatingConstructorAccessorImpl.java:45) at
> java.lang.reflect.Constructor.newInstance(Constructor.java:526) at
> org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
> at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
> at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:327)
> at org.apache.hadoop.hbase.client.ScannerCallable.
> openScanner(ScannerCallable.java:402) at org.apache.hadoop.hbase.
> client.ScannerCallable.call(ScannerCallable.java:203) at
> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:64)
> at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
> at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$
> RetryingRPC.call(ScannerCallableWithReplicas.java:360) at
> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$
> RetryingRPC.call(ScannerCallableWithReplicas.java:334) at
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
> ... 4 moreCaused by: org.apache.hadoop.hbase.ipc.
> RemoteWithExtrasException(org.apache.hadoop.hbase.NotServingRegionException):
> org.apache.hadoop.hbase.NotServingRegionException: Region
> kylin_metadata,,1488427068259.599efd68884035a133f306657bafb309. is not
> online on hf1-cdh1,60020,1488505545946 at org.apache.hadoop.hbase.
> regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2920)
> at org.apache.hadoop.hbase.regionserver.RSRpcServices.
> getRegion(RSRpcServices.java:1053) at org.apache.hadoop.hbase.
> regionserver.RSRpcServices.scan(RSRpcServices.java:2385) at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.
> callBlockingMethod(ClientProtos.java:33648) at
> org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170) at
> org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109) at
> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
> at java.lang.Thread.run(Thread.java:745)
> at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1268)
> at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
> at org.apache.hadoop.hbase.ipc.AbstractRpcClient$
> BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
> at org.apache.hadoop.hbase.protobuf.generated.
> ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094)
> at org.apache.hadoop.hbase.client.ScannerCallable.
> openScanner(ScannerCallable.java:394) ... 10 more
> Could you give me a help, my kylin service can not to use!!!!
--