You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@tez.apache.org by "zhengchenyu (Jira)" <ji...@apache.org> on 2023/06/20 08:11:00 UTC
[jira] [Updated] (TEZ-4497) CartesianProduct run failed in unpartitioned mode
[ https://issues.apache.org/jira/browse/TEZ-4497?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
zhengchenyu updated TEZ-4497:
-----------------------------
Description:
CartesianProduct run failed in unpartitioned mode.
execute command:
{code:java}
hadoop jar ${TEZ_HOME}/tez-examples-0.9.2.jar cartesianproduct -unpartitioned ${inputPath1} ${inputPath2} ${inputPath3} ${outputPath} {code}
error log:
{code:java}
Error: Error while running task ( failure ) : java.lang.IndexOutOfBoundsException
at java.nio.Buffer.checkIndex(Buffer.java:544)
at java.nio.ByteBufferAsLongBufferB.get(ByteBufferAsLongBufferB.java:114)
at org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord.getIndex(TezSpillRecord.java:103)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.getTezIndexRecord(Fetcher.java:739)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.doLocalDiskFetch(Fetcher.java:658)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.setupLocalDiskFetch(Fetcher.java:632)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:280)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:76)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
, errorMessage=Fetch failed:java.lang.IndexOutOfBoundsException
at java.nio.Buffer.checkIndex(Buffer.java:544)
at java.nio.ByteBufferAsLongBufferB.get(ByteBufferAsLongBufferB.java:114)
at org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord.getIndex(TezSpillRecord.java:103)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.getTezIndexRecord(Fetcher.java:739)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.doLocalDiskFetch(Fetcher.java:658)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.setupLocalDiskFetch(Fetcher.java:632)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:280)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:76)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) {code}
Reason:
In my example, if unpartitioned is enable, will use FairCartesianProductVertexManager,
numPartitions is 31 (1000^0.5), it is dismatched with the partition 1 which is specific by UnorderedKVOutput.
was:
CartesianProduct run failed in unpartitioned mode.
execute command:
{code:java}
hadoop jar ${TEZ_HOME}/tez-examples-0.9.2.jar cartesianproduct -unpartitioned ${inputPath1} ${inputPath2} ${inputPath3} ${outputPath} {code}
error log:
{code:java}
Error: Error while running task ( failure ) : java.lang.IndexOutOfBoundsException
at java.nio.Buffer.checkIndex(Buffer.java:544)
at java.nio.ByteBufferAsLongBufferB.get(ByteBufferAsLongBufferB.java:114)
at org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord.getIndex(TezSpillRecord.java:103)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.getTezIndexRecord(Fetcher.java:739)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.doLocalDiskFetch(Fetcher.java:658)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.setupLocalDiskFetch(Fetcher.java:632)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:280)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:76)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
, errorMessage=Fetch failed:java.lang.IndexOutOfBoundsException
at java.nio.Buffer.checkIndex(Buffer.java:544)
at java.nio.ByteBufferAsLongBufferB.get(ByteBufferAsLongBufferB.java:114)
at org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord.getIndex(TezSpillRecord.java:103)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.getTezIndexRecord(Fetcher.java:739)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.doLocalDiskFetch(Fetcher.java:658)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.setupLocalDiskFetch(Fetcher.java:632)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:280)
at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:76)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) {code}
Reason:
In my example, if unpartitioned is enable, will use FairCartesianProductVertexManager,
numPartitions is 31 (1000^0.5), it is dismatched with the partition 1 which is specific by UnorderedKVOutput.
.
> CartesianProduct run failed in unpartitioned mode
> -------------------------------------------------
>
> Key: TEZ-4497
> URL: https://issues.apache.org/jira/browse/TEZ-4497
> Project: Apache Tez
> Issue Type: Bug
> Reporter: zhengchenyu
> Assignee: zhengchenyu
> Priority: Minor
>
> CartesianProduct run failed in unpartitioned mode.
> execute command:
>
> {code:java}
> hadoop jar ${TEZ_HOME}/tez-examples-0.9.2.jar cartesianproduct -unpartitioned ${inputPath1} ${inputPath2} ${inputPath3} ${outputPath} {code}
> error log:
> {code:java}
> Error: Error while running task ( failure ) : java.lang.IndexOutOfBoundsException
> at java.nio.Buffer.checkIndex(Buffer.java:544)
> at java.nio.ByteBufferAsLongBufferB.get(ByteBufferAsLongBufferB.java:114)
> at org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord.getIndex(TezSpillRecord.java:103)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.getTezIndexRecord(Fetcher.java:739)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.doLocalDiskFetch(Fetcher.java:658)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.setupLocalDiskFetch(Fetcher.java:632)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:280)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:76)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
> at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
> at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> , errorMessage=Fetch failed:java.lang.IndexOutOfBoundsException
> at java.nio.Buffer.checkIndex(Buffer.java:544)
> at java.nio.ByteBufferAsLongBufferB.get(ByteBufferAsLongBufferB.java:114)
> at org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord.getIndex(TezSpillRecord.java:103)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.getTezIndexRecord(Fetcher.java:739)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.doLocalDiskFetch(Fetcher.java:658)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.setupLocalDiskFetch(Fetcher.java:632)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:280)
> at org.apache.tez.runtime.library.common.shuffle.Fetcher.callInternal(Fetcher.java:76)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
> at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
> at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748) {code}
> Reason:
> In my example, if unpartitioned is enable, will use FairCartesianProductVertexManager,
> numPartitions is 31 (1000^0.5), it is dismatched with the partition 1 which is specific by UnorderedKVOutput.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)