You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@drill.apache.org by "Boaz Ben-Zvi (JIRA)" <ji...@apache.org> on 2018/07/12 01:38:00 UTC
[jira] [Commented] (DRILL-6590) DATA_WRITE ERROR: Hash Join failed
to write to output file: /tmp/drill/spill/24bac407
[ https://issues.apache.org/jira/browse/DRILL-6590?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16540997#comment-16540997 ]
Boaz Ben-Zvi commented on DRILL-6590:
-------------------------------------
This bug came out of DRILL-6453, where a query got (in an unexplained way) cancelled. It looks like the cancellation somehow caused the spill writing to fail - maybe another thread closed that spill file, while the original thread was writing ?
> DATA_WRITE ERROR: Hash Join failed to write to output file: /tmp/drill/spill/24bac407
> -------------------------------------------------------------------------------------
>
> Key: DRILL-6590
> URL: https://issues.apache.org/jira/browse/DRILL-6590
> Project: Apache Drill
> Issue Type: Bug
> Components: Execution - Flow
> Affects Versions: 1.14.0
> Reporter: Khurram Faraaz
> Priority: Major
>
> Apache Drill 1.14.0 git.commit.id.abbrev=eb946b0
> There was enough space on /tmp, however Hash Join failed to write to spill file
> [test@qa102-45 drill-1.14.0]# clush -a df -h /tmp
> <IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
> <IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 150G 351G 30% /
> <IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
> <IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 17G 484G 4% /
> <IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
> <IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 14G 487G 3% /
> <IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
> <IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 13G 488G 3% /
> Stack trace from drillbit.log
> {noformat}
> 2018-07-10 18:17:51,953 [BitServer-10] WARN o.a.d.exec.rpc.control.WorkEventBus - A fragment message arrived but there was no registered listener for that message: profile {
> state: FAILED
> error {
> error_id: "6e258de2-2d4f-4b48-967d-df1b329955cd"
> endpoint {
> address: "qa102-48.qa.lab"
> user_port: 31010
> control_port: 31011
> data_port: 31012
> version: "1.14.0-SNAPSHOT"
> state: STARTUP
> }
> error_type: DATA_WRITE
> message: "DATA_WRITE ERROR: Hash Join failed to write to output file: /tmp/drill/spill/24bac407-2adb-5763-ed08-cb5714dca2c0_HashJoin_4-22-53/spill15_outer\n\nFragment 4:53\n\n[Error Id: 6e258de2-2d4f-4b48-967d-df1b329955cd on qa102-48.qa.lab:31010]"
> exception {
> exception_class: "java.nio.channels.ClosedByInterruptException"
> stack_trace {
> class_name: "..."
> line_number: 0
> method_name: "..."
> is_native_method: false
> }
> stack_trace {
> class_name: "com.google.protobuf.CodedOutputStream"
> file_name: "CodedOutputStream.java"
> line_number: 833
> method_name: "refreshBuffer"
> is_native_method: false
> }
> stack_trace {
> class_name: "com.google.protobuf.CodedOutputStream"
> file_name: "CodedOutputStream.java"
> line_number: 843
> method_name: "flush"
> is_native_method: false
> }
> stack_trace {
> class_name: "com.google.protobuf.AbstractMessageLite"
> file_name: "AbstractMessageLite.java"
> line_number: 91
> method_name: "writeDelimitedTo"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.cache.VectorSerializer$Writer"
> file_name: "VectorSerializer.java"
> line_number: 97
> method_name: "write"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
> file_name: "HashPartition.java"
> line_number: 346
> method_name: "spillThisPartition"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
> file_name: "HashPartition.java"
> line_number: 263
> method_name: "completeABatch"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
> file_name: "HashPartition.java"
> line_number: 237
> method_name: "completeAnOuterBatch"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
> file_name: "HashPartition.java"
> line_number: 232
> method_name: "appendOuterRow"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.test.generated.HashJoinProbeGen49"
> file_name: "HashJoinProbeTemplate.java"
> line_number: 306
> method_name: "executeProbePhase"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.test.generated.HashJoinProbeGen49"
> file_name: "HashJoinProbeTemplate.java"
> line_number: 393
> method_name: "probeAndProject"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 357
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 172
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 274
> method_name: "sniffNonEmptyBatch"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 236
> method_name: "prefetchFirstBatchFromBothSides"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 216
> method_name: "buildSchema"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 152
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 274
> method_name: "sniffNonEmptyBatch"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 236
> method_name: "prefetchFirstBatchFromBothSides"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 216
> method_name: "buildSchema"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 152
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 274
> method_name: "sniffNonEmptyBatch"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 236
> method_name: "prefetchFirstBatchFromBothSides"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 216
> method_name: "buildSchema"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 152
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 274
> method_name: "sniffNonEmptyBatch"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 236
> method_name: "prefetchFirstBatchFromBothSides"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 216
> method_name: "buildSchema"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 152
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 274
> method_name: "sniffNonEmptyBatch"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 236
> method_name: "prefetchFirstBatchFromBothSides"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 216
> method_name: "buildSchema"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 152
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 109
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
> file_name: "AbstractUnaryRecordBatch.java"
> line_number: 63
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.project.ProjectRecordBatch"
> file_name: "ProjectRecordBatch.java"
> line_number: 147
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 172
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 109
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
> file_name: "AbstractUnaryRecordBatch.java"
> line_number: 63
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 172
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 109
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
> file_name: "AbstractUnaryRecordBatch.java"
> line_number: 63
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 172
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 274
> method_name: "sniffNonEmptyBatch"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 236
> method_name: "prefetchFirstBatchFromBothSides"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 216
> method_name: "buildSchema"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 152
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 274
> method_name: "sniffNonEmptyBatch"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 236
> method_name: "prefetchFirstBatchFromBothSides"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
> file_name: "HashJoinBatch.java"
> line_number: 216
> method_name: "buildSchema"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 152
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 109
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
> file_name: "AbstractUnaryRecordBatch.java"
> line_number: 63
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.project.ProjectRecordBatch"
> file_name: "ProjectRecordBatch.java"
> line_number: 147
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 172
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 109
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.aggregate.HashAggBatch"
> file_name: "HashAggBatch.java"
> line_number: 199
> method_name: "buildSchema"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 152
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 119
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 109
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
> file_name: "AbstractUnaryRecordBatch.java"
> line_number: 63
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.project.ProjectRecordBatch"
> file_name: "ProjectRecordBatch.java"
> line_number: 147
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
> file_name: "AbstractRecordBatch.java"
> line_number: 172
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.BaseRootExec"
> file_name: "BaseRootExec.java"
> line_number: 103
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.SingleSenderCreator$SingleSenderRootExec"
> file_name: "SingleSenderCreator.java"
> line_number: 93
> method_name: "innerNext"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.physical.impl.BaseRootExec"
> file_name: "BaseRootExec.java"
> line_number: 93
> method_name: "next"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.work.fragment.FragmentExecutor$1"
> file_name: "FragmentExecutor.java"
> line_number: 294
> method_name: "run"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.work.fragment.FragmentExecutor$1"
> file_name: "FragmentExecutor.java"
> line_number: 281
> method_name: "run"
> is_native_method: false
> }
> stack_trace {
> class_name: "..."
> line_number: 0
> method_name: "..."
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.hadoop.security.UserGroupInformation"
> file_name: "UserGroupInformation.java"
> line_number: 1595
> method_name: "doAs"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.exec.work.fragment.FragmentExecutor"
> file_name: "FragmentExecutor.java"
> line_number: 281
> method_name: "run"
> is_native_method: false
> }
> stack_trace {
> class_name: "org.apache.drill.common.SelfCleaningRunnable"
> file_name: "SelfCleaningRunnable.java"
> line_number: 38
> method_name: "run"
> is_native_method: false
> }
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)