You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by GitBox <gi...@apache.org> on 2019/04/02 08:08:00 UTC
[GitHub] [incubator-iceberg] aokolnychyi commented on issue #104:
ManifestReader is not properly closed in BaseTableScan
aokolnychyi commented on issue #104: ManifestReader is not properly closed in BaseTableScan
URL: https://github.com/apache/incubator-iceberg/issues/104#issuecomment-478890258
I also see this exception from time to time in `TestParquetWrite`.
```
[Finalizer] WARN org.apache.iceberg.hadoop.HadoopStreams - Unclosed input stream created by:
org.apache.iceberg.hadoop.HadoopStreams$HadoopSeekableInputStream.<init>(HadoopStreams.java:77)
org.apache.iceberg.hadoop.HadoopStreams.wrap(HadoopStreams.java:52)
org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:125)
org.apache.iceberg.avro.AvroIterable.newFileReader(AvroIterable.java:95)
org.apache.iceberg.avro.AvroIterable.iterator(AvroIterable.java:77)
com.google.common.collect.Iterables$6.iterator(Iterables.java:585)
com.google.common.collect.Iterables$8.iterator(Iterables.java:709)
org.apache.iceberg.ManifestReader.iterator(ManifestReader.java:236)
org.apache.iceberg.FilteredManifest.iterator(FilteredManifest.java:128)
com.google.common.collect.Iterables$8.iterator(Iterables.java:709)
com.google.common.collect.Iterables$3.transform(Iterables.java:508)
com.google.common.collect.Iterables$3.transform(Iterables.java:505)
com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48)
com.google.common.collect.Iterators$5.hasNext(Iterators.java:543)
com.google.common.collect.TransformedIterator.hasNext(TransformedIterator.java:43)
com.google.common.collect.TransformedIterator.hasNext(TransformedIterator.java:43)
com.google.common.collect.Iterators$5.hasNext(Iterators.java:542)
org.apache.iceberg.util.BinPacking$PackingIterator.hasNext(BinPacking.java:99)
org.apache.iceberg.io.CloseableIterable$4$1.hasNext(CloseableIterable.java:92)
com.google.common.collect.Iterators.addAll(Iterators.java:356)
com.google.common.collect.Lists.newArrayList(Lists.java:147)
com.google.common.collect.Lists.newArrayList(Lists.java:129)
org.apache.iceberg.spark.source.Reader.tasks(Reader.java:226)
org.apache.iceberg.spark.source.Reader.planInputPartitions(Reader.java:148)
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec.partitions$lzycompute(DataSourceV2ScanExec.scala:76)
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec.partitions(DataSourceV2ScanExec.scala:75)
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec.outputPartitioning(DataSourceV2ScanExec.scala:65)
org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$org$apache$spark$sql$execution$exchange$EnsureRequirements$$ensureDistributionAndOrdering$1.apply(EnsureRequirements.scala:150)
org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$org$apache$spark$sql$execution$exchange$EnsureRequirements$$ensureDistributionAndOrdering$1.apply(EnsureRequirements.scala:149)
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
scala.collection.immutable.List.foreach(List.scala:392)
scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
scala.collection.immutable.List.map(List.scala:296)
org.apache.spark.sql.execution.exchange.EnsureRequirements.org$apache$spark$sql$execution$exchange$EnsureRequirements$$ensureDistributionAndOrdering(EnsureRequirements.scala:149)
org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$apply$1.applyOrElse(EnsureRequirements.scala:304)
org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$apply$1.applyOrElse(EnsureRequirements.scala:296)
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:278)
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:278)
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:277)
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:275)
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:275)
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:326)
org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:324)
org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:275)
org.apache.spark.sql.execution.exchange.EnsureRequirements.apply(EnsureRequirements.scala:296)
org.apache.spark.sql.execution.exchange.EnsureRequirements.apply(EnsureRequirements.scala:38)
org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:87)
org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:87)
scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:124)
scala.collection.immutable.List.foldLeft(List.scala:84)
org.apache.spark.sql.execution.QueryExecution.prepareForExecution(QueryExecution.scala:87)
org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:77)
org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:77)
org.apache.spark.sql.Dataset.withAction(Dataset.scala:3360)
org.apache.spark.sql.Dataset.collectAsList(Dataset.scala:2794)
org.apache.iceberg.spark.source.TestParquetWrite.testBasicWrite(TestParquetWrite.java:98)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
org.junit.rules.RunRules.evaluate(RunRules.java:20)
org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
org.junit.runners.ParentRunner.run(ParentRunner.java:363)
org.junit.runner.JUnitCore.run(JUnitCore.java:137)
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org