You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@iotdb.apache.org by "Jialin Qiao (Jira)" <ji...@apache.org> on 2022/11/24 14:23:00 UTC
[jira] [Assigned] (IOTDB-4508) [cluster]Got an OOM error when using bm with batch=100
[ https://issues.apache.org/jira/browse/IOTDB-4508?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Jialin Qiao reassigned IOTDB-4508:
----------------------------------
Assignee: Haonan Hou (was: Jinrui Zhang)
> [cluster]Got an OOM error when using bm with batch=100
> ------------------------------------------------------
>
> Key: IOTDB-4508
> URL: https://issues.apache.org/jira/browse/IOTDB-4508
> Project: Apache IoTDB
> Issue Type: Bug
> Components: Core/Cluster
> Affects Versions: 0.14.0-SNAPSHOT
> Reporter: FengQingxin
> Assignee: Haonan Hou
> Priority: Major
> Attachments: config.properties, confignode-env.sh, datanode-env.sh, image-2022-09-23-09-11-54-951.png, image-2022-10-13-08-52-13-345.png, iotdb-confignode.properties, iotdb-datanode.properties, log-all-20221012.0.log.gz, log-all-20221012.1.log.gz, log-error-20221012.0.log.gz, logs_Chaos-22.tar.gz, logs_Chaos-23.tar.gz, logs_Chaos-24.tar.gz
>
>
> commit 7dc11bae567cb6aac18827b5c746e84538538ac3
> Author: ZhangHongYin <46...@users.noreply.github.com>
> Date: Thu Sep 22 23:36:17 2022 +0800
> IOTDB-4493 Fix null message in TSStatus (#7402)
> 【oom was happened on172.20.70.24、dump files are on 172.20.70.17 D:\IOTDB-4508】
> Reproduce steps：
> # Setup a cluster with 3C3D({color:#de350b}MultiLeaderConsensus{color})
> # Using 3BMs to insert data(Batch=100)
> # PS: There is no oom errr if we changed benchmark config from 'Batch=100' to 'Batch=10'.
> BM ----- 》 IoTDB Node
> 172.20.70.7 ----- 》 172.20.70.22
> 172.20.70.8 ----- 》 172.20.70.23
> 172.20.70.9 ----- 》 172.20.70.24
>
> 2022-09-23 09:08:23,470 [pool-27-IoTDB-LogDispatcher-DataRegion[10]-2] ERROR o.a.i.c.c.t.WrappedThreadPoolExecutor:104 - Exception in thread pool org.apache.iotdb.threadpool:type=LogDispatcher-DataRegion[10]
> java.lang.OutOfMemoryError: Java heap space
> at java.util.Arrays.copyOf(Arrays.java:3236)
> at java.io.ByteArrayOutputStream.grow(ByteArrayOutputStream.java:118)
> at java.io.ByteArrayOutputStream.ensureCapacity(ByteArrayOutputStream.java:93)
> at java.io.ByteArrayOutputStream.write(ByteArrayOutputStream.java:153)
> at org.apache.thrift.transport.TMemoryBuffer.write(TMemoryBuffer.java:88)
> at org.apache.thrift.protocol.TBinaryProtocol.writeBinary(TBinaryProtocol.java:235)
> at org.apache.iotdb.consensus.multileader.thrift.TLogBatch$TLogBatchStandardScheme.write(TLogBatch.java:524)
> at org.apache.iotdb.consensus.multileader.thrift.TLogBatch$TLogBatchStandardScheme.write(TLogBatch.java:465)
> at org.apache.iotdb.consensus.multileader.thrift.TLogBatch.write(TLogBatch.java:404)
> at org.apache.iotdb.consensus.multileader.thrift.TSyncLogReq$TSyncLogReqStandardScheme.write(TSyncLogReq.java:569)
> at org.apache.iotdb.consensus.multileader.thrift.TSyncLogReq$TSyncLogReqStandardScheme.write(TSyncLogReq.java:490)
> at org.apache.iotdb.consensus.multileader.thrift.TSyncLogReq.write(TSyncLogReq.java:416)
> at org.apache.iotdb.consensus.multileader.thrift.MultiLeaderConsensusIService$syncLog_args$syncLog_argsStandardScheme.write(MultiLeaderConsensusIService.java:563)
> at org.apache.iotdb.consensus.multileader.thrift.MultiLeaderConsensusIService$syncLog_args$syncLog_argsStandardScheme.write(MultiLeaderConsensusIService.java:525)
> at org.apache.iotdb.consensus.multileader.thrift.MultiLeaderConsensusIService$syncLog_args.write(MultiLeaderConsensusIService.java:476)
> at org.apache.iotdb.consensus.multileader.thrift.MultiLeaderConsensusIService$AsyncClient$syncLog_call.write_args(MultiLeaderConsensusIService.java:104)
> at org.apache.thrift.async.TAsyncMethodCall.prepareMethodCall(TAsyncMethodCall.java:129)
> at org.apache.thrift.async.TAsyncClientManager.call(TAsyncClientManager.java:55)
> at org.apache.iotdb.consensus.multileader.thrift.MultiLeaderConsensusIService$AsyncClient.syncLog(MultiLeaderConsensusIService.java:90)
> at org.apache.iotdb.consensus.multileader.logdispatcher.LogDispatcher$LogDispatcherThread.sendBatchAsync(LogDispatcher.java:337)
> at org.apache.iotdb.consensus.multileader.logdispatcher.LogDispatcher$LogDispatcherThread.run(LogDispatcher.java:214)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 2022-09-23 09:08:23,478 [pool-22-IoTDB-ClientRPC-Processor-37$20220923_010343_02955_5] ERROR o.a.i.d.m.p.s.FragmentInstanceDispatcherImpl:129 - [DispatchFailed]
> java.lang.OutOfMemoryError: GC overhead limit exceeded
> at org.apache.iotdb.tsfile.utils.BytesUtils.longToBytes(BytesUtils.java:485)
> at org.apache.iotdb.tsfile.utils.BytesUtils.longToBytes(BytesUtils.java:470)
> at org.apache.iotdb.tsfile.utils.ReadWriteIOUtils.write(ReadWriteIOUtils.java:311)
> at org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode.serializeColumn(InsertTabletNode.java:641)
> at org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode.writeValues(InsertTabletNode.java:582)
> at org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode.subSerialize(InsertTabletNode.java:446)
> at org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode.serializeAttributes(InsertTabletNode.java:427)
> at org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode.serialize(PlanNode.java:111)
> at org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode.serializeToByteBuffer(PlanNode.java:132)
> at org.apache.iotdb.consensus.common.request.IndexedConsensusRequest.lambda$buildSerializedRequests$0(IndexedConsensusRequest.java:71)
> at org.apache.iotdb.consensus.common.request.IndexedConsensusRequest$$Lambda$928/745418928.accept(Unknown Source)
> at java.util.Collections$SingletonList.forEach(Collections.java:4822)
> at org.apache.iotdb.consensus.common.request.IndexedConsensusRequest.buildSerializedRequests(IndexedConsensusRequest.java:71)
> at org.apache.iotdb.consensus.multileader.logdispatcher.LogDispatcher.offer(LogDispatcher.java:112)
> at org.apache.iotdb.consensus.multileader.MultiLeaderServerImpl.write(MultiLeaderServerImpl.java:158)
> at org.apache.iotdb.consensus.multileader.MultiLeaderConsensus.write(MultiLeaderConsensus.java:148)
> at org.apache.iotdb.db.mpp.plan.scheduler.FragmentInstanceDispatcherImpl.dispatchLocally(FragmentInstanceDispatcherImpl.java:280)
> at org.apache.iotdb.db.mpp.plan.scheduler.FragmentInstanceDispatcherImpl.dispatchOneInstance(FragmentInstanceDispatcherImpl.java:143)
> at org.apache.iotdb.db.mpp.plan.scheduler.FragmentInstanceDispatcherImpl.dispatchWriteSync(FragmentInstanceDispatcherImpl.java:125)
> at org.apache.iotdb.db.mpp.plan.scheduler.FragmentInstanceDispatcherImpl.dispatch(FragmentInstanceDispatcherImpl.java:96)
> at org.apache.iotdb.db.mpp.plan.scheduler.ClusterScheduler.start(ClusterScheduler.java:102)
> at org.apache.iotdb.db.mpp.plan.execution.QueryExecution.schedule(QueryExecution.java:264)
> at org.apache.iotdb.db.mpp.plan.execution.QueryExecution.start(QueryExecution.java:184)
> at org.apache.iotdb.db.mpp.plan.Coordinator.execute(Coordinator.java:146)
> at org.apache.iotdb.db.mpp.plan.Coordinator.execute(Coordinator.java:160)
> !image-2022-09-23-09-11-54-951.png!
--
This message was sent by Atlassian Jira
(v8.20.10#820010)