You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@kylin.apache.org by 高扬02 <ga...@bianfeng.com> on 2019/07/16 02:22:20 UTC
Please help:error of hbase coprocessor
Hello,
Kylin 2.6.0,CDH5.10.2
Hbase regionserver error as the picture below.
Please help!
Best regards!
[cid:image004.png@01D53BC0.598166F0]
2019-07-16 10:16:50,268 INFO org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService: start query 8f29e7df-9296-b0e8-f32b-77078b1a8fca in thread RpcServer.FifoWFPBQ.default.handler=20,queue=2,port=60020
2019-07-16 10:16:50,269 ERROR org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService: java.io.IOException: Could not seek StoreFileScanner[HFileScanner for reader reader=hdfs://kylincluster/hbase/data/default/KYLIN_Q81YSFZ5VZ/0c8cd0070e95a704d2abbeeefeb9ad70/F2/448e037f03ba4ab88cd78c1c97e8b6d8, compression=none, cacheConf=blockCache=LruBlockCache{blockCount=2129, currentSize=394673280, freeSize=8527056768, maxSize=8921730048, heapSize=394673280, minSize=8475643392, minFactor=0.95, multiSize=4237821696, multiFactor=0.5, singleSize=2118910848, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, firstKey=\x00\x03\x00\x00\x00\x00\x00\x03\x00\x08\x00\x00\x0BAM\xFF/F2:M/0/Put, lastKey=\x00\x03\x00\x00\x00\x00\x00\x03\xFF\xFF\xFF\xFF\x0BAM\xFF\xFF\x00\x00\x00w\xFF\x01\x7F\x7F\xFF\x01\xC0\x02\x9E\xFF\xFF\x13\x0C\x8C\x0F\xE1z\x80\x03\xC4c\x00/F2:M/0/Put, avgKeyLen=50, avgValueLen=124, entries=3079715, length=421331966, cur=null] to key \x00\x03\x00\x00\x00\x00\x00\x03\xFF\xFF\x01}\x0BAO\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00/F2:/LATEST_TIMESTAMP/DeleteFamily/vlen=0/seqid=0
java.io.IOException: Could not seek StoreFileScanner[HFileScanner for reader reader=hdfs://kylincluster/hbase/data/default/KYLIN_Q81YSFZ5VZ/0c8cd0070e95a704d2abbeeefeb9ad70/F2/448e037f03ba4ab88cd78c1c97e8b6d8, compression=none, cacheConf=blockCache=LruBlockCache{blockCount=2129, currentSize=394673280, freeSize=8527056768, maxSize=8921730048, heapSize=394673280, minSize=8475643392, minFactor=0.95, multiSize=4237821696, multiFactor=0.5, singleSize=2118910848, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, firstKey=\x00\x03\x00\x00\x00\x00\x00\x03\x00\x08\x00\x00\x0BAM\xFF/F2:M/0/Put, lastKey=\x00\x03\x00\x00\x00\x00\x00\x03\xFF\xFF\xFF\xFF\x0BAM\xFF\xFF\x00\x00\x00w\xFF\x01\x7F\x7F\xFF\x01\xC0\x02\x9E\xFF\xFF\x13\x0C\x8C\x0F\xE1z\x80\x03\xC4c\x00/F2:M/0/Put, avgKeyLen=50, avgValueLen=124, entries=3079715, length=421331966, cur=null] to key \x00\x03\x00\x00\x00\x00\x00\x03\xFF\xFF\x01}\x0BAO\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00/F2:/LATEST_TIMESTAMP/DeleteFamily/vlen=0/seqid=0
at org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:217)
at org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:350)
at org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:199)
at org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:2106)
at org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:2096)
at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:5556)
at org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:2578)
at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2564)
at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:2545)
at org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService.visitCube(CubeVisitService.java:283)
at org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos$CubeVisitService.callMethod(CubeVisitProtos.java:5555)
at org.apache.hadoop.hbase.regionserver.HRegion.execService(HRegion.java:7856)
at org.apache.hadoop.hbase.regionserver.RSRpcServices.execServiceOnRegion(RSRpcServices.java:1988)
at org.apache.hadoop.hbase.regionserver.RSRpcServices.execService(RSRpcServices.java:1970)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33652)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2171)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:185)
at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:165)
Caused by: java.io.IOException: On-disk size without header provided is 413048, but block header contains 0. Block offset: 406424097, data starts with: \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00
at org.apache.hadoop.hbase.io.hfile.HFileBlock.validateOnDiskSizeWithoutHeader(HFileBlock.java:526)
at org.apache.hadoop.hbase.io.hfile.HFileBlock.access$700(HFileBlock.java:92)
at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1699)
at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1542)
at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:445)
at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:266)
at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:642)
at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:592)
at org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:294)
at org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:199)
... 18 more
2019-07-16 10:17:10,171 INFO org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService: start query 8f29e7df-9296-b0e8-f32b-77078b1a8fca in thread RpcServer.FifoWFPBQ.default.handler=20,queue=2,port=60020
2019-07-16 10:17:10,171 INFO org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService: Deadline has passed, abort now!
2019-07-16 10:20:38,555 INFO org.apache.hadoop.hbase.io.hfile.LruBlockCache: totalSize=376.39 MB, freeSize=7.94 GB, max=8.31 GB, blockCount=2129, accesses=105060, hits=31477, hitRatio=29.96%, , cachingAccesses=36892, cachingHits=30228, cachingHitsRatio=81.94%, evictions=6869, evicted=1078, evictedPerRun=0.15693695843219757
发件人: 高扬02 [mailto:gaoyang02@bianfeng.com]
发送时间: 2019年7月15日 16:41
收件人: user@kylin.apache.org
主题: error of hbase coprocessor
Hello,
I have a problem with an UHC dimension count_distinct measure error.
The sql is “SELECT data_day,
app_id,
count(distinct bmid_user) AS pep
FROM wl_sets_kylin_hour
WHERE data_day>=date'2019-07-08'
AND data_day< date'2019-07-15'
AND app_id IN ('7003_900000',
'7003_20078',
'7003_20072',
'7003_20036',
'7003_20043',
'7003_20080',
'7003_20041')
GROUP BY data_day,
app_id
order by pep desc”, and it has an error.
Please tell me the direction of error resolution or optimization.
Thank you very much.
[cid:image005.png@01D53BC0.598166F0]
[cid:image006.png@01D53BC0.598166F0]
[cid:image007.png@01D53BC0.598166F0]
2019-07-15 16:34:41,379 ERROR [http-bio-7070-exec-1] controller.BasicController:63 :
org.apache.kylin.rest.exception.InternalErrorException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Coprocessor passed deadline! Maybe server is overloaded at org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService.checkDeadline(CubeVisitService.java:226) at org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService.visitCube(CubeVisitService.java:261) at org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos$CubeVisitService.callMethod(CubeVisitProtos.java:5555) at org.apache.hadoop.hbase.regionserver.HRegion.execService(HRegion.java:7856) at org.apache.hadoop.hbase.regionserver.RSRpcServices.execServiceOnRegion(RSRpcServices.java:1988) at org.apache.hadoop.hbase.regionserver.RSRpcServices.execService(RSRpcServices.java:1970) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:33652) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2171) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:185) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:165)
while executing SQL: "SELECT data_day, app_id, count(distinct bmid_user) AS pep FROM wl_sets_kylin_hour WHERE data_day>=date'2019-07-08' AND data_day< date'2019-07-15' AND app_id IN ('7003_900000', '7003_20078', '7003_20072', '7003_20036', '7003_20043', '7003_20080', '7003_20041') GROUP BY data_day, app_id order by pep desc"
at org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:441)
at org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:367)
at org.apache.kylin.rest.controller.QueryController.query(QueryController.java:87)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:317)
at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:127)
at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:91)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:114)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:137)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:111)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:170)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:63)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilterInternal(BasicAuthenticationFilter.java:215)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:200)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:116)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.header.HeaderWriterFilter.doFilterInternal(HeaderWriterFilter.java:64)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.context.request.async.WebAsyncManagerIntegrationFilter.doFilterInternal(WebAsyncManagerIntegrationFilter.java:56)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:105)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:214)
at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:177)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:198)
at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:244)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:494)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:1025)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:445)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1137)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
2019-07-15 16:35:22,877 INFO [BadQueryDetector] service.BadQueryDetector:147 : Detect bad query.