You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@drill.apache.org by "Reza Baktash (JIRA)" <ji...@apache.org> on 2018/10/16 08:53:00 UTC
[jira] [Commented] (DRILL-6079) Memory leak caused by
ParquetRowGroupScan
[ https://issues.apache.org/jira/browse/DRILL-6079?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16651338#comment-16651338 ]
Reza Baktash commented on DRILL-6079:
-------------------------------------
Apparently your commit was merged into drill-1.14.0 which I am using now. But I get the same error.
I query a bunch of parquet files on hdfs and some of queries fail due to same error as yours. Any idea why this happens?
Some error logs are below:
{noformat}
[Error Id: 40d6908a-a6bf-4ae3-8769-281c345ad9d8 on server3:31010]
at org.apache.drill.exec.rpc.user.QueryResultHandler.resultArrived(QueryResultHandler.java:123) [drill-java-exec-1.14.0.jar:1.14.0]
at org.apache.drill.exec.rpc.user.UserClient.handle(UserClient.java:422) [drill-java-exec-1.14.0.jar:1.14.0]
at org.apache.drill.exec.rpc.user.UserClient.handle(UserClient.java:96) [drill-java-exec-1.14.0.jar:1.14.0]
at org.apache.drill.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:274) [drill-rpc-1.14.0.jar:1.14.0]
at org.apache.drill.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:244) [drill-rpc-1.14.0.jar:1.14.0]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:88) [netty-codec-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:287) [netty-handler-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:312) [netty-codec-4.0.48.Final.jar:4.0.48.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:286) [netty-codec-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1294) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:911) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459) [netty-transport-4.0.48.Final.jar:4.0.48.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131) [netty-common-4.0.48.Final.jar:4.0.48.Final]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_162]
2018-10-16 04:10:00,637 [Client-1] INFO o.a.d.j.i.DrillCursor$ResultsListener - [#19] Query failed:
org.apache.drill.common.exceptions.UserRemoteException: SYSTEM ERROR: IllegalStateException: Memory was leaked by query. Memory leaked: (524288)
Allocator(op:1:8:3:ParquetRowGroupScan) 1000000/524288/33177600/10000000000 (res/actual/peak/limit)
Fragment 1:8
{noformat}
> Memory leak caused by ParquetRowGroupScan
> -----------------------------------------
>
> Key: DRILL-6079
> URL: https://issues.apache.org/jira/browse/DRILL-6079
> Project: Apache Drill
> Issue Type: Bug
> Components: Storage - Parquet
> Reporter: salim achouche
> Assignee: salim achouche
> Priority: Minor
> Labels: ready-to-commit
> Fix For: 1.13.0
>
>
> Concurrency tests with assertion enabled indicate a memory leak in the Parquet scanner code:
> 2017-10-25 17:28:52,149 [260ed3bc-dbdf-8b4a-66d6-b2bd804e8c74:frag:11:63] ERROR o.a.d.e.w.fragment.FragmentExecutor - SYSTEM ERROR: IllegalStateException: Memory was leaked by query. Memory leaked: (2097152)
> Allocator(op:11:63:6:ParquetRowGroupScan) 1000000/2097152/7348224/10000000000 (res/actual/peak/limit)
> Fragment 11:63
> [Error Id: 5927becb-f000-43db-95b5-93b33064a6fd on mperf113.qa.lab:31010]
> org.apache.drill.common.exceptions.UserException: SYSTEM ERROR: IllegalStateException: Memory was leaked by query. Memory leaked: (2097152)
> Allocator(op:11:63:6:ParquetRowGroupScan) 1000000/2097152/7348224/10000000000 (res/actual/peak/limit)
> Fragment 11:63
> [Error Id: 5927becb-f000-43db-95b5-93b33064a6fd on mperf113.qa.lab:31010]
> at org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:586) ~[drill-common-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.exec.work.fragment.FragmentExecutor.sendFinalState(FragmentExecutor.java:301) [drill-java-exec-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.exec.work.fragment.FragmentExecutor.cleanup(FragmentExecutor.java:160) [drill-java-exec-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:267) [drill-java-exec-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.common.SelfCleaningRunnable.run(SelfCleaningRunnable.java:38) [drill-common-1.11.0-mapr.jar:1.11.0-mapr]
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_121]
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_121]
> at java.lang.Thread.run(Thread.java:745) [na:1.8.0_121]
> Caused by: java.lang.IllegalStateException: Memory was leaked by query. Memory leaked: (2097152)
> Allocator(op:11:63:6:ParquetRowGroupScan) 1000000/2097152/7348224/10000000000 (res/actual/peak/limit)
> at org.apache.drill.exec.memory.BaseAllocator.close(BaseAllocator.java:520) ~[drill-memory-base-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.exec.ops.AbstractOperatorExecContext.close(AbstractOperatorExecContext.java:86) ~[drill-java-exec-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.exec.ops.OperatorContextImpl.close(OperatorContextImpl.java:108) ~[drill-java-exec-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.exec.ops.FragmentContext.suppressingClose(FragmentContext.java:435) ~[drill-java-exec-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.exec.ops.FragmentContext.close(FragmentContext.java:424) ~[drill-java-exec-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.exec.work.fragment.FragmentExecutor.closeOutResources(FragmentExecutor.java:324) [drill-java-exec-1.11.0-mapr.jar:1.11.0-mapr]
> at org.apache.drill.exec.work.fragment.FragmentExecutor.cleanup(FragmentExecutor.java:155) [drill-java-exec-1.11.0-mapr.jar:1.11.0-mapr]
> ... 5 common frames omitted
> 2017-10-25 17:28:52,149 [260ed3bc-dbdf-8b4a-66d6-b2bd804e8c74:frag:6:0] ERROR o.a.d.e.w.fragment.FragmentExecutor - SYSTEM ERROR: IllegalStateException: Memory was leaked by query. Memory leaked: (2097152)
> Allocator(op:6:0:5:ParquetRowGroupScan) 1000000/2097152/19947520/10000000000 (res/actual/peak/limit)
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)