You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@drill.apache.org by "Nitin Pawar (Jira)" <ji...@apache.org> on 2020/01/07 17:04:00 UTC
[jira] [Created] (DRILL-7517) Drill 1.16.0 shuts down frequently
Nitin Pawar created DRILL-7517:
----------------------------------
Summary: Drill 1.16.0 shuts down frequently
Key: DRILL-7517
URL: https://issues.apache.org/jira/browse/DRILL-7517
Project: Apache Drill
Issue Type: Bug
Affects Versions: 1.16.0
Reporter: Nitin Pawar
We see following exception every few hours
Our drillbit cluster queries data from S3.
[Error Id: 1f14f398-fab2-40cd-b14c-389d91c002e8 on 10.0.160.5:31010]
org.apache.drill.common.exceptions.UserException: SYSTEM ERROR: IllegalStateException: Connection pool shut down
Fragment 1:0
Please, refer to logs for more information.
[Error Id: 1f14f398-fab2-40cd-b14c-389d91c002e8 on 10.0.160.5:31010]
at org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:630) ~[drill-common-1.16.0.jar:1.16.0]
at org.apache.drill.exec.work.fragment.FragmentExecutor.sendFinalState(FragmentExecutor.java:363) [drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.work.fragment.FragmentExecutor.cleanup(FragmentExecutor.java:219) [drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:329) [drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.common.SelfCleaningRunnable.run(SelfCleaningRunnable.java:38) [drill-common-1.16.0.jar:1.16.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_222]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_222]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_222]
Caused by: java.lang.IllegalStateException: Connection pool shut down
at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:167) ~[httpcore-4.2.4.jar:4.2.4]
at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:199) ~[httpcore-4.2.4.jar:4.2.4]
at org.apache.http.impl.conn.PoolingClientConnectionManager.requestConnection(PoolingClientConnectionManager.java:188) ~[httpclient-4.2.5.jar:4.2.5]
at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_222]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_222]
at com.amazonaws.http.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:72) ~[aws-java-sdk-1.7.4.jar:na]
at com.amazonaws.http.conn.$Proxy76.requestConnection(Unknown Source) ~[na:na]
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:448) ~[httpclient-4.2.5.jar:4.2.5]
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906) ~[httpclient-4.2.5.jar:4.2.5]
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805) ~[httpclient-4.2.5.jar:4.2.5]
at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:384) ~[aws-java-sdk-1.7.4.jar:na]
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:232) ~[aws-java-sdk-1.7.4.jar:na]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3528) ~[aws-java-sdk-1.7.4.jar:na]
at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:976) ~[aws-java-sdk-1.7.4.jar:na]
at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:956) ~[aws-java-sdk-1.7.4.jar:na]
at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:892) ~[hadoop-aws-2.7.4.jar:na]
at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:77) ~[hadoop-aws-2.7.4.jar:na]
at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1426) ~[hadoop-common-2.7.4.jar:na]
at org.apache.drill.exec.store.StorageStrategy.getNonExistentLocations(StorageStrategy.java:200) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.store.StorageStrategy.createFileAndApply(StorageStrategy.java:134) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.store.text.DrillTextRecordWriter.startNewSchema(DrillTextRecordWriter.java:92) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.store.StringOutputRecordWriter.updateSchema(StringOutputRecordWriter.java:44) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.physical.impl.WriterRecordBatch.setupNewSchema(WriterRecordBatch.java:160) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.physical.impl.WriterRecordBatch.innerNext(WriterRecordBatch.java:108) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:186) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:104) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.physical.impl.SingleSenderCreator$SingleSenderRootExec.innerNext(SingleSenderCreator.java:93) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:94) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:296) ~[drill-java-exec-1.16.0.jar:1.16.0]
at org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:283) ~[drill-java-exec-1.16.0.jar:1.16.0]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.8.0_222]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[na:1.8.0_222]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746) ~[hadoop-common-2.7.4.jar:na]
at org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:283) [drill-java-exec-1.16.0.jar:1.16.0]
... 4 common frames omitted
--
This message was sent by Atlassian Jira
(v8.3.4#803005)